diff options
| -rw-r--r-- | common/broadcast.js | 14 | ||||
| -rw-r--r-- | common/entities.js | 2 | ||||
| -rw-r--r-- | common/indexeddb.js | 115 | ||||
| -rw-r--r-- | test/unit/test_broadcast.py | 4 | ||||
| -rw-r--r-- | test/unit/test_indexeddb.py | 287 |
5 files changed, 357 insertions, 65 deletions
diff --git a/common/broadcast.js b/common/broadcast.js index bc18103..cc11a20 100644 --- a/common/broadcast.js +++ b/common/broadcast.js @@ -59,6 +59,20 @@ function out(sender_conn, channel_name, value) sender_conn.port.postMessage(["broadcast", channel_name, value]); } +/* + * prepare()'d message will be broadcasted if the connection is closed or when + * flush() is called. All messages prepared by given sender will be discarded + * when discard() is called. + * + * Timeout will cause the prepared message to be broadcasted after given number + * of miliseconds unless discard() is called in the meantime. It is mostly + * useful as a fallback in case this connection end gets nuked (e.g. because + * browser window is closed) and onDisconnect event is not dispatched to + * background script's Port object. This is only an issue for browsers as old as + * IceCat 60: https://bugzilla.mozilla.org/show_bug.cgi?id=1392067 + * + * Timeout won't be scheduled at all if its value is given as 0. + */ function prepare(sender_conn, channel_name, value, timeout=5000) { sender_conn.port.postMessage(["prepare", channel_name, value, timeout]); diff --git a/common/entities.js b/common/entities.js index 46836b5..3a1346a 100644 --- a/common/entities.js +++ b/common/entities.js @@ -45,7 +45,7 @@ * Convert ver_str into an array representation, e.g. for ver_str="4.6.13.0" * return [4, 6, 13, 0]. */ -const parse_version = ver_str => ver_str.split(".").map(parseInt); +const parse_version = ver_str => ver_str.split(".").map(n => parseInt(n)); /* * ver is an array of integers. rev is an optional integer. Produce string diff --git a/common/indexeddb.js b/common/indexeddb.js index 96d19b7..1741c91 100644 --- a/common/indexeddb.js +++ b/common/indexeddb.js @@ -45,6 +45,7 @@ * IMPORTS_START * IMPORT initial_data * IMPORT entities + * IMPORT broadcast * IMPORTS_END */ @@ -124,22 +125,43 @@ async function get_db() await _save_items(initial_data.resources, initial_data.mappings, ctx); } - db = opened_db; + if (db) + opened_db.close(); + else + db = opened_db; return db; } +/* Helper function used by make_context(). */ +function reject_discard(context) +{ + broadcast.discard(context.sender); + broadcast.close(context.sender); + context.reject(); +} + +/* Helper function used by make_context(). */ +function resolve_flush(context) +{ + broadcast.close(context.sender); + context.resolve(); +} + /* Helper function used by start_items_transaction() and get_db(). */ function make_context(transaction, files) { - files = files || {}; - const context = {transaction, files, file_uses: {}}; + const sender = broadcast.sender_connection(); + files = files || {}; let resolve, reject; - context.result = new Promise((...cbs) => [resolve, reject] = cbs); + const result = new Promise((...cbs) => [resolve, reject] = cbs); + + const context = + {sender, transaction, resolve, reject, result, files, file_uses: {}}; - context.transaction.oncomplete = resolve; - context.transaction.onerror = reject; + transaction.oncomplete = () => resolve_flush(context); + transaction.onerror = () => reject_discard(context); return context; } @@ -221,13 +243,6 @@ async function finalize_items_transaction(context) return context.result; } -async function with_items_transaction(cb, item_store_names, files={}) -{ - const context = await start_items_transaction(item_store_names, files); - await cb(context); - await finalize_items_transaction(context); -} - /* * How a sample data argument to the function below might look like: * @@ -265,10 +280,10 @@ async function with_items_transaction(cb, item_store_names, files={}) * } * } */ -async function save_items(transaction, data) +async function save_items(data) { - const items_store_names = ["resources", "mappings"]; - const context = start_items_transaction(items_store_names, data.files); + const item_store_names = ["resources", "mappings"]; + const context = await start_items_transaction(item_store_names, data.files); return _save_items(data.resources, data.mappings, context); } @@ -302,6 +317,8 @@ async function save_item(item, context) for (const file_ref of entities.get_files(item)) await incr_file_uses(context, file_ref); + broadcast.prepare(context.sender, `idb_changes_${store_name}`, + item.identifier); await _remove_item(store_name, item.identifier, context, false); await idb_put(context.transaction, store_name, item); } @@ -326,30 +343,78 @@ async function _remove_item(store_name, identifier, context) */ async function remove_item(store_name, identifier, context) { + broadcast.prepare(context.sender, `idb_changes_${store_name}`, identifier); await _remove_item(store_name, identifier, context); await idb_del(context.transaction, store_name, identifier); } -const remove_resource = - (identifier, ctx) => remove_item("resources", identifier, ctx); -const remove_mapping = - (identifier, ctx) => remove_item("mappings", identifier, ctx); +/* Callback used when listening to broadcasts while tracking db changes. */ +async function track_change(tracking, identifier) +{ + const transaction = + (await haketilodb.get()).transaction([tracking.store_name]); + const new_val = await idb_get(transaction, tracking.store_name, identifier); + + tracking.onchange({identifier, new_val}); +} + +/* + * Monitor changes to `store_name` IndexedDB object store. + * + * `store_name` should be either "resources" or "mappings". + * + * `onchange` should be a callback that will be called when an item is added, + * modified or removed from the store. The callback will be passed an object + * representing the change as its first argument. This object will have the + * form: + * { + * identifier: "the identifier of modified resource/mapping", + * new_val: undefined // `undefined` if item removed, item object otherwise + * } + * + * Returns a [tracking, all_current_items] array where `tracking` is an object + * that can be later passed to haketilodb.untrack() to stop tracking changes and + * `all_current_items` is an array of items currently present in the object + * store. + * + * It is possible that `onchange` gets spuriously fired even when an item is not + * actually modified or that it only gets called once after multiple quick + * changes to an item. + */ +async function track(store_name, onchange) +{ + const tracking = {store_name, onchange}; + tracking.listener = + broadcast.listener_connection(msg => track_change(tracking, msg[1])); + broadcast.subscribe(tracking.listener, `idb_changes_${store_name}`); + + const transaction = (await haketilodb.get()).transaction([store_name]); + const all_req = transaction.objectStore(store_name).getAll(); + + return [tracking, (await wait_request(all_req)).target.result]; +} + +function untrack(tracking) +{ + broadcast.close(tracking.listener); +} const haketilodb = { get: get_db, save_items, save_item, - remove_resource, - remove_mapping, + remove_resource: (id, ctx) => remove_item("resources", id, ctx), + remove_mapping: (id, ctx) => remove_item("mappings", id, ctx), start_items_transaction, - finalize_items_transaction + finalize_items_transaction, + track_resources: onchange => track("resources", onchange), + track_mappings: onchange => track("mappings", onchange), + untrack }; /* * EXPORTS_START * EXPORT haketilodb * EXPORT idb_get - * EXPORT idb_put - * EXPORT idb_del * EXPORTS_END */ diff --git a/test/unit/test_broadcast.py b/test/unit/test_broadcast.py index c8c19d1..11a61b0 100644 --- a/test/unit/test_broadcast.py +++ b/test/unit/test_broadcast.py @@ -22,8 +22,8 @@ import pytest from ..script_loader import load_script def broker_js(): - return load_script('background/broadcast_broker.js', - ['common', 'background']) + ';start_broadcast_broker();' + js = load_script('background/broadcast_broker.js', ['common', 'background']) + return js + ';start_broadcast_broker();' def broadcast_js(): return load_script('common/broadcast.js', ['common']) diff --git a/test/unit/test_indexeddb.py b/test/unit/test_indexeddb.py index d48946e..3604ee9 100644 --- a/test/unit/test_indexeddb.py +++ b/test/unit/test_indexeddb.py @@ -18,13 +18,17 @@ Haketilo unit tests - IndexedDB access # CC0 1.0 Universal License for more details. import pytest +import json from hashlib import sha256 +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.common.exceptions import WebDriverException from ..script_loader import load_script -@pytest.fixture(scope="session") -def indexeddb_code(): - yield load_script('common/indexeddb.js', ['common']) +def indexeddb_js(): + return load_script('common/indexeddb.js', ['common']) def sample_file(contents): return { @@ -44,32 +48,69 @@ sample_files = { sample_files_by_hash = dict([[file['hash_key'], file['contents']] for file in sample_files.values()]) +# Sample resource definitions. They'd normally contain more fields but here we +# use simplified versions. + +def make_sample_resource(): + return { + 'source_copyright': [ + file_ref('report.spdx'), + file_ref('LICENSES/somelicense.txt') + ], + 'type': 'resource', + 'identifier': 'helloapple', + 'scripts': [file_ref('hello.js'), file_ref('bye.js')] + } + +def make_sample_mapping(): + return { + 'source_copyright': [ + file_ref('report.spdx'), + file_ref('README.md') + ], + 'type': 'mapping', + 'identifier': 'helloapple' + } + def file_ref(file_name): return {'file': file_name, 'hash_key': sample_files[file_name]['hash_key']} @pytest.mark.get_page('https://gotmyowndoma.in') -def test_save_remove_item(execute_in_page, indexeddb_code): +def test_haketilodb_save_remove(execute_in_page): """ indexeddb.js facilitates operating on Haketilo's internal database. Verify database operations work properly. """ - execute_in_page(indexeddb_code) - # Don't use Haketilo's default initial data. - execute_in_page('initial_data = {};') + execute_in_page(indexeddb_js()) + # Mock some unwanted imports. + execute_in_page( + '''{ + initial_data = {}; + + const broadcast_mock = {}; + const nop = () => {}; + for (const key in broadcast) + broadcast_mock[key] = nop; + broadcast = broadcast_mock; + }''') # Start with no database. execute_in_page( - '''{ + ''' async function delete_db() { - let resolve; - const result = new Promise(_resolve => resolve = _resolve); + if (db) { + db.close(); + db = null; + } + let resolve, reject; + const result = new Promise((...cbs) => [resolve, reject] = cbs); const request = indexedDB.deleteDatabase("haketilo"); - [request.onsuccess, request.onerror] = [resolve, resolve]; + [request.onsuccess, request.onerror] = [resolve, reject]; await result; } returnval(delete_db()); - }''' + ''' ) # Facilitate retrieving all IndexedDB contents. @@ -93,18 +134,8 @@ def test_save_remove_item(execute_in_page, indexeddb_code): } ''') - # Sample resource definition. It'd normally contain more fields but here - # we use a simplified version. - sample_item = { - 'source_copyright': [ - file_ref('report.spdx'), - file_ref('LICENSES/somelicense.txt') - ], - 'type': 'resource', - 'identifier': 'helloapple', - 'scripts': [file_ref('hello.js'), file_ref('bye.js')], - } - next(iter(sample_item['source_copyright']))['ugly_extra_property'] = True + sample_item = make_sample_resource() + sample_item['source_copyright'][0]['extra_prop'] = True database_contents = execute_in_page( '''{ @@ -133,11 +164,8 @@ def test_save_remove_item(execute_in_page, indexeddb_code): sample_item['scripts'].append(file_ref('combined.js')) incomplete_files = {**sample_files_by_hash} incomplete_files.pop(sample_files['combined.js']['hash_key']) - print ('incomplete files:', incomplete_files) - print ('sample item:', sample_item) result = execute_in_page( '''{ - console.log('sample item', arguments[0]); const promise = (async () => { const context = await start_items_transaction(["resources"], arguments[1]); @@ -163,14 +191,7 @@ def test_save_remove_item(execute_in_page, indexeddb_code): == sorted(val, key=keyfun) # See if adding another item that partially uses first's files works OK. - sample_item = { - 'source_copyright': [ - file_ref('report.spdx'), - file_ref('README.md') - ], - 'type': 'mapping', - 'identifier': 'helloapple', - } + sample_item = make_sample_mapping() database_contents = execute_in_page( '''{ const promise = start_items_transaction(["mappings"], arguments[1]) @@ -196,14 +217,16 @@ def test_save_remove_item(execute_in_page, indexeddb_code): assert files == dict([(file['hash_key'], file['contents']) for file in sample_files_list]) - assert database_contents['mappings'] == [sample_item] + del database_contents['resources'][0]['source_copyright'][0]['extra_prop'] + assert database_contents['resources'] == [make_sample_resource()] + assert database_contents['mappings'] == [sample_item] # Try removing the items to get an empty database again. results = [None, None] for i, item_type in enumerate(['resource', 'mapping']): results[i] = execute_in_page( f'''{{ - const remover = remove_{item_type}; + const remover = haketilodb.remove_{item_type}; const promise = start_items_transaction(["{item_type}s"], {{}}) .then(ctx => remover('helloapple', ctx).then(() => ctx)) @@ -229,3 +252,193 @@ def test_save_remove_item(execute_in_page, indexeddb_code): assert results[0]['mappings'] == [sample_item] assert results[1] == dict([(key, []) for key in results[0].keys()]) + + # Try initializing an empty database with sample initial data object. + sample_resource = make_sample_resource() + sample_mapping = make_sample_mapping() + initial_data = { + 'resources': { + 'helloapple': { + '1.12': sample_resource, + '0.9': 'something_that_should_get_ignored', + '1': 'something_that_should_get_ignored', + '1.1': 'something_that_should_get_ignored', + '1.11.1': 'something_that_should_get_ignored', + } + }, + 'mappings': { + 'helloapple': { + '0.1.1': sample_mapping + } + }, + 'files': sample_files_by_hash + } + database_contents = execute_in_page( + ''' + initial_data = arguments[0]; + returnval(delete_db().then(() => get_database_contents())); + ''', + initial_data) + assert database_contents['resources'] == [sample_resource] + assert database_contents['mappings'] == [sample_mapping] + +def broker_js(): + js = load_script('background/broadcast_broker.js', ['common', 'background']) + return js + ';start_broadcast_broker();' + +test_page_html = ''' +<!DOCTYPE html> +<script src="/testpage.js"></script> +<h2>resources</h2> +<ul id="resources"></ul> +<h2>mappings</h2> +<ul id="mappings"></ul> +''' + +@pytest.mark.ext_data({ + 'background_script': broker_js, + 'test_page': test_page_html, + 'extra_files': { + 'testpage.js': indexeddb_js + } +}) +@pytest.mark.usefixtures('webextension') +def test_haketilodb_track(driver, execute_in_page, wait_elem_text): + """ + Verify IndexedDB object change notifications are correctly broadcasted + through extension's background script and allow for object store contents + to be tracked in any execution context. + """ + # Let's open the same extension's test page in a second window. Window 0 + # will be used to make changed to IndexedDB and window 1 to "track" those + # changes. + driver.execute_script('window.open(window.location.href, "_blank");') + windows = [*driver.window_handles] + assert len(windows) == 2 + + # Mock initial_data. + sample_resource = make_sample_resource() + sample_mapping = make_sample_mapping() + initial_data = { + 'resources': { + 'helloapple': { + '1.0': sample_resource + } + }, + 'mappings': { + 'helloapple': { + '0.1.1': sample_mapping + } + }, + 'files': sample_files_by_hash + } + for window in reversed(windows): + driver.switch_to.window(window) + execute_in_page('initial_data = arguments[0];', initial_data) + + # See if haketilodb.track_*() functions properly return the already-existing + # items. + execute_in_page( + ''' + function update_item(store_name, change) + { + console.log('update', ...arguments); + const elem_id = `${store_name}_${change.identifier}`; + let elem = document.getElementById(elem_id); + elem = elem || document.createElement("li"); + elem.id = elem_id; + elem.innerText = JSON.stringify(change.new_val); + document.getElementById(store_name).append(elem); + if (change.new_val === undefined) + elem.remove(); + } + + let resource_tracking, resource_items, mapping_tracking, mapping_items; + + async function start_tracking() + { + const update_resource = change => update_item("resources", change); + const update_mapping = change => update_item("mappings", change); + + [resource_tracking, resource_items] = + await haketilodb.track_resources(update_resource); + [mapping_tracking, mapping_items] = + await haketilodb.track_mappings(update_mapping); + + for (const item of resource_items) + update_resource({identifier: item.identifier, new_val: item}); + for (const item of mapping_items) + update_mapping({identifier: item.identifier, new_val: item}); + } + + returnval(start_tracking()); + ''') + + item_counts = driver.execute_script( + ''' + const childcount = id => document.getElementById(id).childElementCount; + return ["resources", "mappings"].map(childcount); + ''') + assert item_counts == [1, 1] + resource_json = driver.find_element_by_id('resources_helloapple').text + mapping_json = driver.find_element_by_id('mappings_helloapple').text + assert json.loads(resource_json) == sample_resource + assert json.loads(mapping_json) == sample_mapping + + # See if item additions get tracked properly. + driver.switch_to.window(windows[1]) + sample_resource2 = make_sample_resource() + sample_resource2['identifier'] = 'helloapple-copy' + sample_mapping2 = make_sample_mapping() + sample_mapping2['identifier'] = 'helloapple-copy' + sample_data = { + 'resources': { + 'helloapple-copy': { + '1.0': sample_resource2 + } + }, + 'mappings': { + 'helloapple-copy': { + '0.1.1': sample_mapping2 + } + }, + 'files': sample_files_by_hash + } + execute_in_page('returnval(haketilodb.save_items(arguments[0]));', + sample_data) + + driver.switch_to.window(windows[0]) + driver.implicitly_wait(10) + resource_json = driver.find_element_by_id('resources_helloapple-copy').text + mapping_json = driver.find_element_by_id('mappings_helloapple-copy').text + driver.implicitly_wait(0) + assert json.loads(resource_json) == sample_resource2 + assert json.loads(mapping_json) == sample_mapping2 + + # See if item deletions get tracked properly. + driver.switch_to.window(windows[1]) + execute_in_page( + '''{ + async function remove_items() + { + const store_names = ["resources", "mappings"]; + const ctx = await haketilodb.start_items_transaction(store_names, {}); + await haketilodb.remove_resource("helloapple", ctx); + await haketilodb.remove_mapping("helloapple-copy", ctx); + await haketilodb.finalize_items_transaction(ctx); + } + returnval(remove_items()); + }''') + + removed_ids = ['mappings_helloapple-copy', 'resources_helloapple'] + def condition_items_absent(driver): + for id in removed_ids: + try: + driver.find_element_by_id(id) + return False + except WebDriverException: + pass + return True + + driver.switch_to.window(windows[0]) + WebDriverWait(driver, 10).until(condition_items_absent) |