facilitate tracking of IndexedDB item store contents

2 files changed, 252 insertions, 39 deletions

diff --git a/test/unit/test_broadcast.py b/test/unit/test_broadcast.py
index c8c19d1..11a61b0 100644
--- a/test/unit/test_broadcast.py
+++ b/test/unit/test_broadcast.py
@@ -22,8 +22,8 @@ import pytest
 from ..script_loader import load_script
 
 def broker_js():
-    return load_script('background/broadcast_broker.js',
-                       ['common', 'background']) + ';start_broadcast_broker();'
+    js = load_script('background/broadcast_broker.js', ['common', 'background'])
+    return js + ';start_broadcast_broker();'
 
 def broadcast_js():
     return load_script('common/broadcast.js', ['common'])
diff --git a/test/unit/test_indexeddb.py b/test/unit/test_indexeddb.py
index d48946e..3604ee9 100644
--- a/test/unit/test_indexeddb.py
+++ b/test/unit/test_indexeddb.py
@@ -18,13 +18,17 @@ Haketilo unit tests - IndexedDB access
 # CC0 1.0 Universal License for more details.
 
 import pytest
+import json
 from hashlib import sha256
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import WebDriverException
 
 from ..script_loader import load_script
 
-@pytest.fixture(scope="session")
-def indexeddb_code():
-    yield load_script('common/indexeddb.js', ['common'])
+def indexeddb_js():
+    return load_script('common/indexeddb.js', ['common'])
 
 def sample_file(contents):
     return {
@@ -44,32 +48,69 @@ sample_files = {
 sample_files_by_hash = dict([[file['hash_key'], file['contents']]
                              for file in sample_files.values()])
 
+# Sample resource definitions. They'd normally contain more fields but here we
+# use simplified versions.
+
+def make_sample_resource():
+    return {
+        'source_copyright': [
+            file_ref('report.spdx'),
+            file_ref('LICENSES/somelicense.txt')
+        ],
+        'type': 'resource',
+        'identifier': 'helloapple',
+        'scripts': [file_ref('hello.js'), file_ref('bye.js')]
+    }
+
+def make_sample_mapping():
+    return {
+        'source_copyright': [
+            file_ref('report.spdx'),
+            file_ref('README.md')
+        ],
+        'type': 'mapping',
+        'identifier': 'helloapple'
+    }
+
 def file_ref(file_name):
     return {'file': file_name, 'hash_key': sample_files[file_name]['hash_key']}
 
 @pytest.mark.get_page('https://gotmyowndoma.in')
-def test_save_remove_item(execute_in_page, indexeddb_code):
+def test_haketilodb_save_remove(execute_in_page):
     """
     indexeddb.js facilitates operating on Haketilo's internal database.
     Verify database operations work properly.
     """
-    execute_in_page(indexeddb_code)
-    # Don't use Haketilo's default initial data.
-    execute_in_page('initial_data = {};')
+    execute_in_page(indexeddb_js())
+    # Mock some unwanted imports.
+    execute_in_page(
+        '''{
+        initial_data = {};
+
+        const broadcast_mock = {};
+        const nop = () => {};
+        for (const key in broadcast)
+            broadcast_mock[key] = nop;
+        broadcast = broadcast_mock;
+        }''')
 
     # Start with no database.
     execute_in_page(
-        '''{
+        '''
         async function delete_db() {
-            let resolve;
-            const result = new Promise(_resolve => resolve = _resolve);
+            if (db) {
+                db.close();
+                db = null;
+            }
+            let resolve, reject;
+            const result = new Promise((...cbs) => [resolve, reject] = cbs);
             const request = indexedDB.deleteDatabase("haketilo");
-            [request.onsuccess, request.onerror] = [resolve, resolve];
+            [request.onsuccess, request.onerror] = [resolve, reject];
             await result;
         }
 
         returnval(delete_db());
-        }'''
+        '''
     )
 
     # Facilitate retrieving all IndexedDB contents.
@@ -93,18 +134,8 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
         }
         ''')
 
-    # Sample resource definition. It'd normally contain more fields but here
-    # we use a simplified version.
-    sample_item = {
-        'source_copyright': [
-            file_ref('report.spdx'),
-            file_ref('LICENSES/somelicense.txt')
-        ],
-        'type': 'resource',
-        'identifier': 'helloapple',
-        'scripts': [file_ref('hello.js'), file_ref('bye.js')],
-    }
-    next(iter(sample_item['source_copyright']))['ugly_extra_property'] = True
+    sample_item = make_sample_resource()
+    sample_item['source_copyright'][0]['extra_prop'] = True
 
     database_contents = execute_in_page(
         '''{
@@ -133,11 +164,8 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
     sample_item['scripts'].append(file_ref('combined.js'))
     incomplete_files = {**sample_files_by_hash}
     incomplete_files.pop(sample_files['combined.js']['hash_key'])
-    print ('incomplete files:', incomplete_files)
-    print ('sample item:', sample_item)
     result = execute_in_page(
         '''{
-        console.log('sample item', arguments[0]);
         const promise = (async () => {
             const context =
                 await start_items_transaction(["resources"], arguments[1]);
@@ -163,14 +191,7 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
             == sorted(val,                        key=keyfun)
 
     # See if adding another item that partially uses first's files works OK.
-    sample_item = {
-        'source_copyright': [
-            file_ref('report.spdx'),
-            file_ref('README.md')
-        ],
-        'type': 'mapping',
-        'identifier': 'helloapple',
-    }
+    sample_item = make_sample_mapping()
     database_contents = execute_in_page(
         '''{
         const promise = start_items_transaction(["mappings"], arguments[1])
@@ -196,14 +217,16 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
     assert files == dict([(file['hash_key'], file['contents'])
                           for file in sample_files_list])
 
-    assert database_contents['mappings'] == [sample_item]
+    del database_contents['resources'][0]['source_copyright'][0]['extra_prop']
+    assert database_contents['resources'] == [make_sample_resource()]
+    assert database_contents['mappings']  == [sample_item]
 
     # Try removing the items to get an empty database again.
     results = [None, None]
     for i, item_type in enumerate(['resource', 'mapping']):
          results[i] = execute_in_page(
             f'''{{
-            const remover = remove_{item_type};
+            const remover = haketilodb.remove_{item_type};
             const promise =
                 start_items_transaction(["{item_type}s"], {{}})
                 .then(ctx => remover('helloapple', ctx).then(() => ctx))
@@ -229,3 +252,193 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
     assert results[0]['mappings'] == [sample_item]
 
     assert results[1] == dict([(key, []) for key in  results[0].keys()])
+
+    # Try initializing an empty database with sample initial data object.
+    sample_resource = make_sample_resource()
+    sample_mapping = make_sample_mapping()
+    initial_data = {
+        'resources': {
+            'helloapple': {
+                '1.12':   sample_resource,
+                '0.9':    'something_that_should_get_ignored',
+                '1':      'something_that_should_get_ignored',
+                '1.1':    'something_that_should_get_ignored',
+                '1.11.1': 'something_that_should_get_ignored',
+            }
+        },
+        'mappings': {
+            'helloapple': {
+                '0.1.1': sample_mapping
+            }
+        },
+        'files': sample_files_by_hash
+    }
+    database_contents = execute_in_page(
+        '''
+        initial_data = arguments[0];
+        returnval(delete_db().then(() => get_database_contents()));
+        ''',
+        initial_data)
+    assert database_contents['resources'] == [sample_resource]
+    assert database_contents['mappings']  == [sample_mapping]
+
+def broker_js():
+    js = load_script('background/broadcast_broker.js', ['common', 'background'])
+    return js + ';start_broadcast_broker();'
+
+test_page_html = '''
+<!DOCTYPE html>
+<script src="/testpage.js"></script>
+<h2>resources</h2>
+<ul id="resources"></ul>
+<h2>mappings</h2>
+<ul id="mappings"></ul>
+'''
+
+@pytest.mark.ext_data({
+    'background_script': broker_js,
+    'test_page':         test_page_html,
+    'extra_files': {
+        'testpage.js':   indexeddb_js
+    }
+})
+@pytest.mark.usefixtures('webextension')
+def test_haketilodb_track(driver, execute_in_page, wait_elem_text):
+    """
+    Verify IndexedDB object change notifications are correctly broadcasted
+    through extension's background script and allow for object store contents
+    to be tracked in any execution context.
+    """
+    # Let's open the same extension's test page in a second window. Window 0
+    # will be used to make changed to IndexedDB and window 1 to "track" those
+    # changes.
+    driver.execute_script('window.open(window.location.href, "_blank");')
+    windows = [*driver.window_handles]
+    assert len(windows) == 2
+
+    # Mock initial_data.
+    sample_resource = make_sample_resource()
+    sample_mapping = make_sample_mapping()
+    initial_data = {
+        'resources': {
+            'helloapple': {
+                '1.0': sample_resource
+            }
+        },
+        'mappings': {
+            'helloapple': {
+                '0.1.1': sample_mapping
+            }
+        },
+        'files': sample_files_by_hash
+    }
+    for window in reversed(windows):
+        driver.switch_to.window(window)
+        execute_in_page('initial_data = arguments[0];', initial_data)
+
+    # See if haketilodb.track_*() functions properly return the already-existing
+    # items.
+    execute_in_page(
+        '''
+        function update_item(store_name, change)
+        {
+            console.log('update', ...arguments);
+            const elem_id = `${store_name}_${change.identifier}`;
+            let elem = document.getElementById(elem_id);
+            elem = elem || document.createElement("li");
+            elem.id = elem_id;
+            elem.innerText = JSON.stringify(change.new_val);
+            document.getElementById(store_name).append(elem);
+            if (change.new_val === undefined)
+                elem.remove();
+        }
+
+        let resource_tracking, resource_items, mapping_tracking, mapping_items;
+
+        async function start_tracking()
+        {
+            const update_resource = change => update_item("resources", change);
+            const update_mapping  = change => update_item("mappings",  change);
+
+            [resource_tracking, resource_items] =
+                await haketilodb.track_resources(update_resource);
+            [mapping_tracking, mapping_items] =
+                await haketilodb.track_mappings(update_mapping);
+
+            for (const item of resource_items)
+                update_resource({identifier: item.identifier, new_val: item});
+            for (const item of mapping_items)
+                update_mapping({identifier: item.identifier, new_val: item});
+        }
+
+        returnval(start_tracking());
+        ''')
+
+    item_counts = driver.execute_script(
+        '''
+        const childcount = id => document.getElementById(id).childElementCount;
+        return ["resources", "mappings"].map(childcount);
+        ''')
+    assert item_counts == [1, 1]
+    resource_json = driver.find_element_by_id('resources_helloapple').text
+    mapping_json =  driver.find_element_by_id('mappings_helloapple').text
+    assert json.loads(resource_json) == sample_resource
+    assert json.loads(mapping_json)  == sample_mapping
+
+    # See if item additions get tracked properly.
+    driver.switch_to.window(windows[1])
+    sample_resource2 = make_sample_resource()
+    sample_resource2['identifier'] = 'helloapple-copy'
+    sample_mapping2 = make_sample_mapping()
+    sample_mapping2['identifier'] = 'helloapple-copy'
+    sample_data = {
+        'resources': {
+            'helloapple-copy': {
+                '1.0': sample_resource2
+            }
+        },
+        'mappings': {
+            'helloapple-copy': {
+                '0.1.1': sample_mapping2
+            }
+        },
+        'files': sample_files_by_hash
+    }
+    execute_in_page('returnval(haketilodb.save_items(arguments[0]));',
+                    sample_data)
+
+    driver.switch_to.window(windows[0])
+    driver.implicitly_wait(10)
+    resource_json = driver.find_element_by_id('resources_helloapple-copy').text
+    mapping_json =  driver.find_element_by_id('mappings_helloapple-copy').text
+    driver.implicitly_wait(0)
+    assert json.loads(resource_json) == sample_resource2
+    assert json.loads(mapping_json)  == sample_mapping2
+
+    # See if item deletions get tracked properly.
+    driver.switch_to.window(windows[1])
+    execute_in_page(
+        '''{
+        async function remove_items()
+        {
+        const store_names = ["resources", "mappings"];
+        const ctx = await haketilodb.start_items_transaction(store_names, {});
+        await haketilodb.remove_resource("helloapple", ctx);
+        await haketilodb.remove_mapping("helloapple-copy", ctx);
+        await haketilodb.finalize_items_transaction(ctx);
+        }
+        returnval(remove_items());
+        }''')
+
+    removed_ids = ['mappings_helloapple-copy', 'resources_helloapple']
+    def condition_items_absent(driver):
+        for id in removed_ids:
+            try:
+                driver.find_element_by_id(id)
+                return False
+            except WebDriverException:
+                pass
+        return True
+
+    driver.switch_to.window(windows[0])
+    WebDriverWait(driver, 10).until(condition_items_absent)