facilitate tracking of IndexedDB item store contents

5 files changed, 357 insertions, 65 deletions

diff --git a/common/broadcast.js b/common/broadcast.js
index bc18103..cc11a20 100644
--- a/common/broadcast.js
+++ b/common/broadcast.js
@@ -59,6 +59,20 @@ function out(sender_conn, channel_name, value)
     sender_conn.port.postMessage(["broadcast", channel_name, value]);
 }
 
+/*
+ * prepare()'d message will be broadcasted if the connection is closed or when
+ * flush() is called. All messages prepared by given sender will be discarded
+ * when discard() is called.
+ *
+ * Timeout will cause the prepared message to be broadcasted after given number
+ * of miliseconds unless discard() is called in the meantime. It is mostly
+ * useful as a fallback in case this connection end gets nuked (e.g. because
+ * browser window is closed) and onDisconnect event is not dispatched to
+ * background script's Port object. This is only an issue for browsers as old as
+ * IceCat 60: https://bugzilla.mozilla.org/show_bug.cgi?id=1392067
+ *
+ * Timeout won't be scheduled at all if its value is given as 0.
+ */
 function prepare(sender_conn, channel_name, value, timeout=5000)
 {
     sender_conn.port.postMessage(["prepare", channel_name, value, timeout]);
diff --git a/common/entities.js b/common/entities.js
index 46836b5..3a1346a 100644
--- a/common/entities.js
+++ b/common/entities.js
@@ -45,7 +45,7 @@
  * Convert ver_str into an array representation, e.g. for ver_str="4.6.13.0"
  * return [4, 6, 13, 0].
  */
-const parse_version = ver_str => ver_str.split(".").map(parseInt);
+const parse_version = ver_str => ver_str.split(".").map(n => parseInt(n));
 
 /*
  * ver is an array of integers. rev is an optional integer. Produce string
diff --git a/common/indexeddb.js b/common/indexeddb.js
index 96d19b7..1741c91 100644
--- a/common/indexeddb.js
+++ b/common/indexeddb.js
@@ -45,6 +45,7 @@
  * IMPORTS_START
  * IMPORT initial_data
  * IMPORT entities
+ * IMPORT broadcast
  * IMPORTS_END
  */
 
@@ -124,22 +125,43 @@ async function get_db()
 	await _save_items(initial_data.resources, initial_data.mappings, ctx);
     }
 
-    db = opened_db;
+    if (db)
+	opened_db.close();
+    else
+	db = opened_db;
 
     return db;
 }
 
+/* Helper function used by make_context(). */
+function reject_discard(context)
+{
+    broadcast.discard(context.sender);
+    broadcast.close(context.sender);
+    context.reject();
+}
+
+/* Helper function used by make_context(). */
+function resolve_flush(context)
+{
+    broadcast.close(context.sender);
+    context.resolve();
+}
+
 /* Helper function used by start_items_transaction() and get_db(). */
 function make_context(transaction, files)
 {
-    files = files || {};
-    const context = {transaction, files, file_uses: {}};
+    const sender = broadcast.sender_connection();
 
+    files = files || {};
     let resolve, reject;
-    context.result = new Promise((...cbs) => [resolve, reject] = cbs);
+    const result = new Promise((...cbs) => [resolve, reject] = cbs);
+
+    const context =
+	  {sender, transaction, resolve, reject, result, files, file_uses: {}};
 
-    context.transaction.oncomplete = resolve;
-    context.transaction.onerror = reject;
+    transaction.oncomplete = () => resolve_flush(context);
+    transaction.onerror = () => reject_discard(context);
 
     return context;
 }
@@ -221,13 +243,6 @@ async function finalize_items_transaction(context)
     return context.result;
 }
 
-async function with_items_transaction(cb, item_store_names, files={})
-{
-    const context = await start_items_transaction(item_store_names, files);
-    await cb(context);
-    await finalize_items_transaction(context);
-}
-
 /*
  * How a sample data argument to the function below might look like:
  *
@@ -265,10 +280,10 @@ async function with_items_transaction(cb, item_store_names, files={})
  *     }
  * }
  */
-async function save_items(transaction, data)
+async function save_items(data)
 {
-    const items_store_names = ["resources", "mappings"];
-    const context = start_items_transaction(items_store_names, data.files);
+    const item_store_names = ["resources", "mappings"];
+    const context = await start_items_transaction(item_store_names, data.files);
 
     return _save_items(data.resources, data.mappings, context);
 }
@@ -302,6 +317,8 @@ async function save_item(item, context)
     for (const file_ref of entities.get_files(item))
 	await incr_file_uses(context, file_ref);
 
+    broadcast.prepare(context.sender, `idb_changes_${store_name}`,
+		      item.identifier);
     await _remove_item(store_name, item.identifier, context, false);
     await idb_put(context.transaction, store_name, item);
 }
@@ -326,30 +343,78 @@ async function _remove_item(store_name, identifier, context)
  */
 async function remove_item(store_name, identifier, context)
 {
+    broadcast.prepare(context.sender, `idb_changes_${store_name}`, identifier);
     await _remove_item(store_name, identifier, context);
     await idb_del(context.transaction, store_name, identifier);
 }
 
-const remove_resource =
-      (identifier, ctx) => remove_item("resources", identifier, ctx);
-const remove_mapping =
-      (identifier, ctx) => remove_item("mappings",  identifier, ctx);
+/* Callback used when listening to broadcasts while tracking db changes. */
+async function track_change(tracking, identifier)
+{
+    const transaction =
+	  (await haketilodb.get()).transaction([tracking.store_name]);
+    const new_val = await idb_get(transaction, tracking.store_name, identifier);
+
+    tracking.onchange({identifier, new_val});
+}
+
+/*
+ * Monitor changes to `store_name` IndexedDB object store.
+ *
+ * `store_name` should be either "resources" or "mappings".
+ *
+ * `onchange` should be a callback that will be called when an item is added,
+ * modified or removed from the store. The callback will be passed an object
+ * representing the change as its first argument. This object will have the
+ * form:
+ * {
+ *     identifier: "the identifier of modified resource/mapping",
+ *     new_val: undefined // `undefined` if item removed, item object otherwise
+ * }
+ *
+ * Returns a [tracking, all_current_items] array where `tracking` is an object
+ * that can be later passed to haketilodb.untrack() to stop tracking changes and
+ * `all_current_items` is an array of items currently present in the object
+ * store.
+ *
+ * It is possible that `onchange` gets spuriously fired even when an item is not
+ * actually modified or that it only gets called once after multiple quick
+ * changes to an item.
+ */
+async function track(store_name, onchange)
+{
+    const tracking = {store_name, onchange};
+    tracking.listener =
+	broadcast.listener_connection(msg => track_change(tracking, msg[1]));
+    broadcast.subscribe(tracking.listener, `idb_changes_${store_name}`);
+
+    const transaction = (await haketilodb.get()).transaction([store_name]);
+    const all_req = transaction.objectStore(store_name).getAll();
+
+    return [tracking, (await wait_request(all_req)).target.result];
+}
+
+function untrack(tracking)
+{
+    broadcast.close(tracking.listener);
+}
 
 const haketilodb = {
     get: get_db,
     save_items,
     save_item,
-    remove_resource,
-    remove_mapping,
+    remove_resource: (id, ctx) => remove_item("resources", id, ctx),
+    remove_mapping:  (id, ctx) => remove_item("mappings",  id, ctx),
     start_items_transaction,
-    finalize_items_transaction
+    finalize_items_transaction,
+    track_resources: onchange => track("resources", onchange),
+    track_mappings:  onchange => track("mappings",  onchange),
+    untrack
 };
 
 /*
  * EXPORTS_START
  * EXPORT haketilodb
  * EXPORT idb_get
- * EXPORT idb_put
- * EXPORT idb_del
  * EXPORTS_END
  */
diff --git a/test/unit/test_broadcast.py b/test/unit/test_broadcast.py
index c8c19d1..11a61b0 100644
--- a/test/unit/test_broadcast.py
+++ b/test/unit/test_broadcast.py
@@ -22,8 +22,8 @@ import pytest
 from ..script_loader import load_script
 
 def broker_js():
-    return load_script('background/broadcast_broker.js',
-                       ['common', 'background']) + ';start_broadcast_broker();'
+    js = load_script('background/broadcast_broker.js', ['common', 'background'])
+    return js + ';start_broadcast_broker();'
 
 def broadcast_js():
     return load_script('common/broadcast.js', ['common'])
diff --git a/test/unit/test_indexeddb.py b/test/unit/test_indexeddb.py
index d48946e..3604ee9 100644
--- a/test/unit/test_indexeddb.py
+++ b/test/unit/test_indexeddb.py
@@ -18,13 +18,17 @@ Haketilo unit tests - IndexedDB access
 # CC0 1.0 Universal License for more details.
 
 import pytest
+import json
 from hashlib import sha256
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import WebDriverException
 
 from ..script_loader import load_script
 
-@pytest.fixture(scope="session")
-def indexeddb_code():
-    yield load_script('common/indexeddb.js', ['common'])
+def indexeddb_js():
+    return load_script('common/indexeddb.js', ['common'])
 
 def sample_file(contents):
     return {
@@ -44,32 +48,69 @@ sample_files = {
 sample_files_by_hash = dict([[file['hash_key'], file['contents']]
                              for file in sample_files.values()])
 
+# Sample resource definitions. They'd normally contain more fields but here we
+# use simplified versions.
+
+def make_sample_resource():
+    return {
+        'source_copyright': [
+            file_ref('report.spdx'),
+            file_ref('LICENSES/somelicense.txt')
+        ],
+        'type': 'resource',
+        'identifier': 'helloapple',
+        'scripts': [file_ref('hello.js'), file_ref('bye.js')]
+    }
+
+def make_sample_mapping():
+    return {
+        'source_copyright': [
+            file_ref('report.spdx'),
+            file_ref('README.md')
+        ],
+        'type': 'mapping',
+        'identifier': 'helloapple'
+    }
+
 def file_ref(file_name):
     return {'file': file_name, 'hash_key': sample_files[file_name]['hash_key']}
 
 @pytest.mark.get_page('https://gotmyowndoma.in')
-def test_save_remove_item(execute_in_page, indexeddb_code):
+def test_haketilodb_save_remove(execute_in_page):
     """
     indexeddb.js facilitates operating on Haketilo's internal database.
     Verify database operations work properly.
     """
-    execute_in_page(indexeddb_code)
-    # Don't use Haketilo's default initial data.
-    execute_in_page('initial_data = {};')
+    execute_in_page(indexeddb_js())
+    # Mock some unwanted imports.
+    execute_in_page(
+        '''{
+        initial_data = {};
+
+        const broadcast_mock = {};
+        const nop = () => {};
+        for (const key in broadcast)
+            broadcast_mock[key] = nop;
+        broadcast = broadcast_mock;
+        }''')
 
     # Start with no database.
     execute_in_page(
-        '''{
+        '''
         async function delete_db() {
-            let resolve;
-            const result = new Promise(_resolve => resolve = _resolve);
+            if (db) {
+                db.close();
+                db = null;
+            }
+            let resolve, reject;
+            const result = new Promise((...cbs) => [resolve, reject] = cbs);
             const request = indexedDB.deleteDatabase("haketilo");
-            [request.onsuccess, request.onerror] = [resolve, resolve];
+            [request.onsuccess, request.onerror] = [resolve, reject];
             await result;
         }
 
         returnval(delete_db());
-        }'''
+        '''
     )
 
     # Facilitate retrieving all IndexedDB contents.
@@ -93,18 +134,8 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
         }
         ''')
 
-    # Sample resource definition. It'd normally contain more fields but here
-    # we use a simplified version.
-    sample_item = {
-        'source_copyright': [
-            file_ref('report.spdx'),
-            file_ref('LICENSES/somelicense.txt')
-        ],
-        'type': 'resource',
-        'identifier': 'helloapple',
-        'scripts': [file_ref('hello.js'), file_ref('bye.js')],
-    }
-    next(iter(sample_item['source_copyright']))['ugly_extra_property'] = True
+    sample_item = make_sample_resource()
+    sample_item['source_copyright'][0]['extra_prop'] = True
 
     database_contents = execute_in_page(
         '''{
@@ -133,11 +164,8 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
     sample_item['scripts'].append(file_ref('combined.js'))
     incomplete_files = {**sample_files_by_hash}
     incomplete_files.pop(sample_files['combined.js']['hash_key'])
-    print ('incomplete files:', incomplete_files)
-    print ('sample item:', sample_item)
     result = execute_in_page(
         '''{
-        console.log('sample item', arguments[0]);
         const promise = (async () => {
             const context =
                 await start_items_transaction(["resources"], arguments[1]);
@@ -163,14 +191,7 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
             == sorted(val,                        key=keyfun)
 
     # See if adding another item that partially uses first's files works OK.
-    sample_item = {
-        'source_copyright': [
-            file_ref('report.spdx'),
-            file_ref('README.md')
-        ],
-        'type': 'mapping',
-        'identifier': 'helloapple',
-    }
+    sample_item = make_sample_mapping()
     database_contents = execute_in_page(
         '''{
         const promise = start_items_transaction(["mappings"], arguments[1])
@@ -196,14 +217,16 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
     assert files == dict([(file['hash_key'], file['contents'])
                           for file in sample_files_list])
 
-    assert database_contents['mappings'] == [sample_item]
+    del database_contents['resources'][0]['source_copyright'][0]['extra_prop']
+    assert database_contents['resources'] == [make_sample_resource()]
+    assert database_contents['mappings']  == [sample_item]
 
     # Try removing the items to get an empty database again.
     results = [None, None]
     for i, item_type in enumerate(['resource', 'mapping']):
          results[i] = execute_in_page(
             f'''{{
-            const remover = remove_{item_type};
+            const remover = haketilodb.remove_{item_type};
             const promise =
                 start_items_transaction(["{item_type}s"], {{}})
                 .then(ctx => remover('helloapple', ctx).then(() => ctx))
@@ -229,3 +252,193 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
     assert results[0]['mappings'] == [sample_item]
 
     assert results[1] == dict([(key, []) for key in  results[0].keys()])
+
+    # Try initializing an empty database with sample initial data object.
+    sample_resource = make_sample_resource()
+    sample_mapping = make_sample_mapping()
+    initial_data = {
+        'resources': {
+            'helloapple': {
+                '1.12':   sample_resource,
+                '0.9':    'something_that_should_get_ignored',
+                '1':      'something_that_should_get_ignored',
+                '1.1':    'something_that_should_get_ignored',
+                '1.11.1': 'something_that_should_get_ignored',
+            }
+        },
+        'mappings': {
+            'helloapple': {
+                '0.1.1': sample_mapping
+            }
+        },
+        'files': sample_files_by_hash
+    }
+    database_contents = execute_in_page(
+        '''
+        initial_data = arguments[0];
+        returnval(delete_db().then(() => get_database_contents()));
+        ''',
+        initial_data)
+    assert database_contents['resources'] == [sample_resource]
+    assert database_contents['mappings']  == [sample_mapping]
+
+def broker_js():
+    js = load_script('background/broadcast_broker.js', ['common', 'background'])
+    return js + ';start_broadcast_broker();'
+
+test_page_html = '''
+<!DOCTYPE html>
+<script src="/testpage.js"></script>
+<h2>resources</h2>
+<ul id="resources"></ul>
+<h2>mappings</h2>
+<ul id="mappings"></ul>
+'''
+
+@pytest.mark.ext_data({
+    'background_script': broker_js,
+    'test_page':         test_page_html,
+    'extra_files': {
+        'testpage.js':   indexeddb_js
+    }
+})
+@pytest.mark.usefixtures('webextension')
+def test_haketilodb_track(driver, execute_in_page, wait_elem_text):
+    """
+    Verify IndexedDB object change notifications are correctly broadcasted
+    through extension's background script and allow for object store contents
+    to be tracked in any execution context.
+    """
+    # Let's open the same extension's test page in a second window. Window 0
+    # will be used to make changed to IndexedDB and window 1 to "track" those
+    # changes.
+    driver.execute_script('window.open(window.location.href, "_blank");')
+    windows = [*driver.window_handles]
+    assert len(windows) == 2
+
+    # Mock initial_data.
+    sample_resource = make_sample_resource()
+    sample_mapping = make_sample_mapping()
+    initial_data = {
+        'resources': {
+            'helloapple': {
+                '1.0': sample_resource
+            }
+        },
+        'mappings': {
+            'helloapple': {
+                '0.1.1': sample_mapping
+            }
+        },
+        'files': sample_files_by_hash
+    }
+    for window in reversed(windows):
+        driver.switch_to.window(window)
+        execute_in_page('initial_data = arguments[0];', initial_data)
+
+    # See if haketilodb.track_*() functions properly return the already-existing
+    # items.
+    execute_in_page(
+        '''
+        function update_item(store_name, change)
+        {
+            console.log('update', ...arguments);
+            const elem_id = `${store_name}_${change.identifier}`;
+            let elem = document.getElementById(elem_id);
+            elem = elem || document.createElement("li");
+            elem.id = elem_id;
+            elem.innerText = JSON.stringify(change.new_val);
+            document.getElementById(store_name).append(elem);
+            if (change.new_val === undefined)
+                elem.remove();
+        }
+
+        let resource_tracking, resource_items, mapping_tracking, mapping_items;
+
+        async function start_tracking()
+        {
+            const update_resource = change => update_item("resources", change);
+            const update_mapping  = change => update_item("mappings",  change);
+
+            [resource_tracking, resource_items] =
+                await haketilodb.track_resources(update_resource);
+            [mapping_tracking, mapping_items] =
+                await haketilodb.track_mappings(update_mapping);
+
+            for (const item of resource_items)
+                update_resource({identifier: item.identifier, new_val: item});
+            for (const item of mapping_items)
+                update_mapping({identifier: item.identifier, new_val: item});
+        }
+
+        returnval(start_tracking());
+        ''')
+
+    item_counts = driver.execute_script(
+        '''
+        const childcount = id => document.getElementById(id).childElementCount;
+        return ["resources", "mappings"].map(childcount);
+        ''')
+    assert item_counts == [1, 1]
+    resource_json = driver.find_element_by_id('resources_helloapple').text
+    mapping_json =  driver.find_element_by_id('mappings_helloapple').text
+    assert json.loads(resource_json) == sample_resource
+    assert json.loads(mapping_json)  == sample_mapping
+
+    # See if item additions get tracked properly.
+    driver.switch_to.window(windows[1])
+    sample_resource2 = make_sample_resource()
+    sample_resource2['identifier'] = 'helloapple-copy'
+    sample_mapping2 = make_sample_mapping()
+    sample_mapping2['identifier'] = 'helloapple-copy'
+    sample_data = {
+        'resources': {
+            'helloapple-copy': {
+                '1.0': sample_resource2
+            }
+        },
+        'mappings': {
+            'helloapple-copy': {
+                '0.1.1': sample_mapping2
+            }
+        },
+        'files': sample_files_by_hash
+    }
+    execute_in_page('returnval(haketilodb.save_items(arguments[0]));',
+                    sample_data)
+
+    driver.switch_to.window(windows[0])
+    driver.implicitly_wait(10)
+    resource_json = driver.find_element_by_id('resources_helloapple-copy').text
+    mapping_json =  driver.find_element_by_id('mappings_helloapple-copy').text
+    driver.implicitly_wait(0)
+    assert json.loads(resource_json) == sample_resource2
+    assert json.loads(mapping_json)  == sample_mapping2
+
+    # See if item deletions get tracked properly.
+    driver.switch_to.window(windows[1])
+    execute_in_page(
+        '''{
+        async function remove_items()
+        {
+        const store_names = ["resources", "mappings"];
+        const ctx = await haketilodb.start_items_transaction(store_names, {});
+        await haketilodb.remove_resource("helloapple", ctx);
+        await haketilodb.remove_mapping("helloapple-copy", ctx);
+        await haketilodb.finalize_items_transaction(ctx);
+        }
+        returnval(remove_items());
+        }''')
+
+    removed_ids = ['mappings_helloapple-copy', 'resources_helloapple']
+    def condition_items_absent(driver):
+        for id in removed_ids:
+            try:
+                driver.find_element_by_id(id)
+                return False
+            except WebDriverException:
+                pass
+        return True
+
+    driver.switch_to.window(windows[0])
+    WebDriverWait(driver, 10).until(condition_items_absent)