summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorWojtek Kosior <koszko@koszko.org>2021-12-16 14:37:09 +0100
committerWojtek Kosior <koszko@koszko.org>2021-12-16 14:37:09 +0100
commitb7378a9994724750198e0d165c575be8538334fb (patch)
tree1dd4a9252869591e85d21d5eb3e31c70a0fa0938
parent9a7623de1458f799baa109d0afbed08547874550 (diff)
downloadbrowser-extension-b7378a9994724750198e0d165c575be8538334fb.tar.gz
browser-extension-b7378a9994724750198e0d165c575be8538334fb.zip
facilitate tracking of IndexedDB item store contents
-rw-r--r--common/broadcast.js14
-rw-r--r--common/entities.js2
-rw-r--r--common/indexeddb.js115
-rw-r--r--test/unit/test_broadcast.py4
-rw-r--r--test/unit/test_indexeddb.py287
5 files changed, 357 insertions, 65 deletions
diff --git a/common/broadcast.js b/common/broadcast.js
index bc18103..cc11a20 100644
--- a/common/broadcast.js
+++ b/common/broadcast.js
@@ -59,6 +59,20 @@ function out(sender_conn, channel_name, value)
sender_conn.port.postMessage(["broadcast", channel_name, value]);
}
+/*
+ * prepare()'d message will be broadcasted if the connection is closed or when
+ * flush() is called. All messages prepared by given sender will be discarded
+ * when discard() is called.
+ *
+ * Timeout will cause the prepared message to be broadcasted after given number
+ * of miliseconds unless discard() is called in the meantime. It is mostly
+ * useful as a fallback in case this connection end gets nuked (e.g. because
+ * browser window is closed) and onDisconnect event is not dispatched to
+ * background script's Port object. This is only an issue for browsers as old as
+ * IceCat 60: https://bugzilla.mozilla.org/show_bug.cgi?id=1392067
+ *
+ * Timeout won't be scheduled at all if its value is given as 0.
+ */
function prepare(sender_conn, channel_name, value, timeout=5000)
{
sender_conn.port.postMessage(["prepare", channel_name, value, timeout]);
diff --git a/common/entities.js b/common/entities.js
index 46836b5..3a1346a 100644
--- a/common/entities.js
+++ b/common/entities.js
@@ -45,7 +45,7 @@
* Convert ver_str into an array representation, e.g. for ver_str="4.6.13.0"
* return [4, 6, 13, 0].
*/
-const parse_version = ver_str => ver_str.split(".").map(parseInt);
+const parse_version = ver_str => ver_str.split(".").map(n => parseInt(n));
/*
* ver is an array of integers. rev is an optional integer. Produce string
diff --git a/common/indexeddb.js b/common/indexeddb.js
index 96d19b7..1741c91 100644
--- a/common/indexeddb.js
+++ b/common/indexeddb.js
@@ -45,6 +45,7 @@
* IMPORTS_START
* IMPORT initial_data
* IMPORT entities
+ * IMPORT broadcast
* IMPORTS_END
*/
@@ -124,22 +125,43 @@ async function get_db()
await _save_items(initial_data.resources, initial_data.mappings, ctx);
}
- db = opened_db;
+ if (db)
+ opened_db.close();
+ else
+ db = opened_db;
return db;
}
+/* Helper function used by make_context(). */
+function reject_discard(context)
+{
+ broadcast.discard(context.sender);
+ broadcast.close(context.sender);
+ context.reject();
+}
+
+/* Helper function used by make_context(). */
+function resolve_flush(context)
+{
+ broadcast.close(context.sender);
+ context.resolve();
+}
+
/* Helper function used by start_items_transaction() and get_db(). */
function make_context(transaction, files)
{
- files = files || {};
- const context = {transaction, files, file_uses: {}};
+ const sender = broadcast.sender_connection();
+ files = files || {};
let resolve, reject;
- context.result = new Promise((...cbs) => [resolve, reject] = cbs);
+ const result = new Promise((...cbs) => [resolve, reject] = cbs);
+
+ const context =
+ {sender, transaction, resolve, reject, result, files, file_uses: {}};
- context.transaction.oncomplete = resolve;
- context.transaction.onerror = reject;
+ transaction.oncomplete = () => resolve_flush(context);
+ transaction.onerror = () => reject_discard(context);
return context;
}
@@ -221,13 +243,6 @@ async function finalize_items_transaction(context)
return context.result;
}
-async function with_items_transaction(cb, item_store_names, files={})
-{
- const context = await start_items_transaction(item_store_names, files);
- await cb(context);
- await finalize_items_transaction(context);
-}
-
/*
* How a sample data argument to the function below might look like:
*
@@ -265,10 +280,10 @@ async function with_items_transaction(cb, item_store_names, files={})
* }
* }
*/
-async function save_items(transaction, data)
+async function save_items(data)
{
- const items_store_names = ["resources", "mappings"];
- const context = start_items_transaction(items_store_names, data.files);
+ const item_store_names = ["resources", "mappings"];
+ const context = await start_items_transaction(item_store_names, data.files);
return _save_items(data.resources, data.mappings, context);
}
@@ -302,6 +317,8 @@ async function save_item(item, context)
for (const file_ref of entities.get_files(item))
await incr_file_uses(context, file_ref);
+ broadcast.prepare(context.sender, `idb_changes_${store_name}`,
+ item.identifier);
await _remove_item(store_name, item.identifier, context, false);
await idb_put(context.transaction, store_name, item);
}
@@ -326,30 +343,78 @@ async function _remove_item(store_name, identifier, context)
*/
async function remove_item(store_name, identifier, context)
{
+ broadcast.prepare(context.sender, `idb_changes_${store_name}`, identifier);
await _remove_item(store_name, identifier, context);
await idb_del(context.transaction, store_name, identifier);
}
-const remove_resource =
- (identifier, ctx) => remove_item("resources", identifier, ctx);
-const remove_mapping =
- (identifier, ctx) => remove_item("mappings", identifier, ctx);
+/* Callback used when listening to broadcasts while tracking db changes. */
+async function track_change(tracking, identifier)
+{
+ const transaction =
+ (await haketilodb.get()).transaction([tracking.store_name]);
+ const new_val = await idb_get(transaction, tracking.store_name, identifier);
+
+ tracking.onchange({identifier, new_val});
+}
+
+/*
+ * Monitor changes to `store_name` IndexedDB object store.
+ *
+ * `store_name` should be either "resources" or "mappings".
+ *
+ * `onchange` should be a callback that will be called when an item is added,
+ * modified or removed from the store. The callback will be passed an object
+ * representing the change as its first argument. This object will have the
+ * form:
+ * {
+ * identifier: "the identifier of modified resource/mapping",
+ * new_val: undefined // `undefined` if item removed, item object otherwise
+ * }
+ *
+ * Returns a [tracking, all_current_items] array where `tracking` is an object
+ * that can be later passed to haketilodb.untrack() to stop tracking changes and
+ * `all_current_items` is an array of items currently present in the object
+ * store.
+ *
+ * It is possible that `onchange` gets spuriously fired even when an item is not
+ * actually modified or that it only gets called once after multiple quick
+ * changes to an item.
+ */
+async function track(store_name, onchange)
+{
+ const tracking = {store_name, onchange};
+ tracking.listener =
+ broadcast.listener_connection(msg => track_change(tracking, msg[1]));
+ broadcast.subscribe(tracking.listener, `idb_changes_${store_name}`);
+
+ const transaction = (await haketilodb.get()).transaction([store_name]);
+ const all_req = transaction.objectStore(store_name).getAll();
+
+ return [tracking, (await wait_request(all_req)).target.result];
+}
+
+function untrack(tracking)
+{
+ broadcast.close(tracking.listener);
+}
const haketilodb = {
get: get_db,
save_items,
save_item,
- remove_resource,
- remove_mapping,
+ remove_resource: (id, ctx) => remove_item("resources", id, ctx),
+ remove_mapping: (id, ctx) => remove_item("mappings", id, ctx),
start_items_transaction,
- finalize_items_transaction
+ finalize_items_transaction,
+ track_resources: onchange => track("resources", onchange),
+ track_mappings: onchange => track("mappings", onchange),
+ untrack
};
/*
* EXPORTS_START
* EXPORT haketilodb
* EXPORT idb_get
- * EXPORT idb_put
- * EXPORT idb_del
* EXPORTS_END
*/
diff --git a/test/unit/test_broadcast.py b/test/unit/test_broadcast.py
index c8c19d1..11a61b0 100644
--- a/test/unit/test_broadcast.py
+++ b/test/unit/test_broadcast.py
@@ -22,8 +22,8 @@ import pytest
from ..script_loader import load_script
def broker_js():
- return load_script('background/broadcast_broker.js',
- ['common', 'background']) + ';start_broadcast_broker();'
+ js = load_script('background/broadcast_broker.js', ['common', 'background'])
+ return js + ';start_broadcast_broker();'
def broadcast_js():
return load_script('common/broadcast.js', ['common'])
diff --git a/test/unit/test_indexeddb.py b/test/unit/test_indexeddb.py
index d48946e..3604ee9 100644
--- a/test/unit/test_indexeddb.py
+++ b/test/unit/test_indexeddb.py
@@ -18,13 +18,17 @@ Haketilo unit tests - IndexedDB access
# CC0 1.0 Universal License for more details.
import pytest
+import json
from hashlib import sha256
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import WebDriverException
from ..script_loader import load_script
-@pytest.fixture(scope="session")
-def indexeddb_code():
- yield load_script('common/indexeddb.js', ['common'])
+def indexeddb_js():
+ return load_script('common/indexeddb.js', ['common'])
def sample_file(contents):
return {
@@ -44,32 +48,69 @@ sample_files = {
sample_files_by_hash = dict([[file['hash_key'], file['contents']]
for file in sample_files.values()])
+# Sample resource definitions. They'd normally contain more fields but here we
+# use simplified versions.
+
+def make_sample_resource():
+ return {
+ 'source_copyright': [
+ file_ref('report.spdx'),
+ file_ref('LICENSES/somelicense.txt')
+ ],
+ 'type': 'resource',
+ 'identifier': 'helloapple',
+ 'scripts': [file_ref('hello.js'), file_ref('bye.js')]
+ }
+
+def make_sample_mapping():
+ return {
+ 'source_copyright': [
+ file_ref('report.spdx'),
+ file_ref('README.md')
+ ],
+ 'type': 'mapping',
+ 'identifier': 'helloapple'
+ }
+
def file_ref(file_name):
return {'file': file_name, 'hash_key': sample_files[file_name]['hash_key']}
@pytest.mark.get_page('https://gotmyowndoma.in')
-def test_save_remove_item(execute_in_page, indexeddb_code):
+def test_haketilodb_save_remove(execute_in_page):
"""
indexeddb.js facilitates operating on Haketilo's internal database.
Verify database operations work properly.
"""
- execute_in_page(indexeddb_code)
- # Don't use Haketilo's default initial data.
- execute_in_page('initial_data = {};')
+ execute_in_page(indexeddb_js())
+ # Mock some unwanted imports.
+ execute_in_page(
+ '''{
+ initial_data = {};
+
+ const broadcast_mock = {};
+ const nop = () => {};
+ for (const key in broadcast)
+ broadcast_mock[key] = nop;
+ broadcast = broadcast_mock;
+ }''')
# Start with no database.
execute_in_page(
- '''{
+ '''
async function delete_db() {
- let resolve;
- const result = new Promise(_resolve => resolve = _resolve);
+ if (db) {
+ db.close();
+ db = null;
+ }
+ let resolve, reject;
+ const result = new Promise((...cbs) => [resolve, reject] = cbs);
const request = indexedDB.deleteDatabase("haketilo");
- [request.onsuccess, request.onerror] = [resolve, resolve];
+ [request.onsuccess, request.onerror] = [resolve, reject];
await result;
}
returnval(delete_db());
- }'''
+ '''
)
# Facilitate retrieving all IndexedDB contents.
@@ -93,18 +134,8 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
}
''')
- # Sample resource definition. It'd normally contain more fields but here
- # we use a simplified version.
- sample_item = {
- 'source_copyright': [
- file_ref('report.spdx'),
- file_ref('LICENSES/somelicense.txt')
- ],
- 'type': 'resource',
- 'identifier': 'helloapple',
- 'scripts': [file_ref('hello.js'), file_ref('bye.js')],
- }
- next(iter(sample_item['source_copyright']))['ugly_extra_property'] = True
+ sample_item = make_sample_resource()
+ sample_item['source_copyright'][0]['extra_prop'] = True
database_contents = execute_in_page(
'''{
@@ -133,11 +164,8 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
sample_item['scripts'].append(file_ref('combined.js'))
incomplete_files = {**sample_files_by_hash}
incomplete_files.pop(sample_files['combined.js']['hash_key'])
- print ('incomplete files:', incomplete_files)
- print ('sample item:', sample_item)
result = execute_in_page(
'''{
- console.log('sample item', arguments[0]);
const promise = (async () => {
const context =
await start_items_transaction(["resources"], arguments[1]);
@@ -163,14 +191,7 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
== sorted(val, key=keyfun)
# See if adding another item that partially uses first's files works OK.
- sample_item = {
- 'source_copyright': [
- file_ref('report.spdx'),
- file_ref('README.md')
- ],
- 'type': 'mapping',
- 'identifier': 'helloapple',
- }
+ sample_item = make_sample_mapping()
database_contents = execute_in_page(
'''{
const promise = start_items_transaction(["mappings"], arguments[1])
@@ -196,14 +217,16 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
assert files == dict([(file['hash_key'], file['contents'])
for file in sample_files_list])
- assert database_contents['mappings'] == [sample_item]
+ del database_contents['resources'][0]['source_copyright'][0]['extra_prop']
+ assert database_contents['resources'] == [make_sample_resource()]
+ assert database_contents['mappings'] == [sample_item]
# Try removing the items to get an empty database again.
results = [None, None]
for i, item_type in enumerate(['resource', 'mapping']):
results[i] = execute_in_page(
f'''{{
- const remover = remove_{item_type};
+ const remover = haketilodb.remove_{item_type};
const promise =
start_items_transaction(["{item_type}s"], {{}})
.then(ctx => remover('helloapple', ctx).then(() => ctx))
@@ -229,3 +252,193 @@ def test_save_remove_item(execute_in_page, indexeddb_code):
assert results[0]['mappings'] == [sample_item]
assert results[1] == dict([(key, []) for key in results[0].keys()])
+
+ # Try initializing an empty database with sample initial data object.
+ sample_resource = make_sample_resource()
+ sample_mapping = make_sample_mapping()
+ initial_data = {
+ 'resources': {
+ 'helloapple': {
+ '1.12': sample_resource,
+ '0.9': 'something_that_should_get_ignored',
+ '1': 'something_that_should_get_ignored',
+ '1.1': 'something_that_should_get_ignored',
+ '1.11.1': 'something_that_should_get_ignored',
+ }
+ },
+ 'mappings': {
+ 'helloapple': {
+ '0.1.1': sample_mapping
+ }
+ },
+ 'files': sample_files_by_hash
+ }
+ database_contents = execute_in_page(
+ '''
+ initial_data = arguments[0];
+ returnval(delete_db().then(() => get_database_contents()));
+ ''',
+ initial_data)
+ assert database_contents['resources'] == [sample_resource]
+ assert database_contents['mappings'] == [sample_mapping]
+
+def broker_js():
+ js = load_script('background/broadcast_broker.js', ['common', 'background'])
+ return js + ';start_broadcast_broker();'
+
+test_page_html = '''
+<!DOCTYPE html>
+<script src="/testpage.js"></script>
+<h2>resources</h2>
+<ul id="resources"></ul>
+<h2>mappings</h2>
+<ul id="mappings"></ul>
+'''
+
+@pytest.mark.ext_data({
+ 'background_script': broker_js,
+ 'test_page': test_page_html,
+ 'extra_files': {
+ 'testpage.js': indexeddb_js
+ }
+})
+@pytest.mark.usefixtures('webextension')
+def test_haketilodb_track(driver, execute_in_page, wait_elem_text):
+ """
+ Verify IndexedDB object change notifications are correctly broadcasted
+ through extension's background script and allow for object store contents
+ to be tracked in any execution context.
+ """
+ # Let's open the same extension's test page in a second window. Window 0
+ # will be used to make changed to IndexedDB and window 1 to "track" those
+ # changes.
+ driver.execute_script('window.open(window.location.href, "_blank");')
+ windows = [*driver.window_handles]
+ assert len(windows) == 2
+
+ # Mock initial_data.
+ sample_resource = make_sample_resource()
+ sample_mapping = make_sample_mapping()
+ initial_data = {
+ 'resources': {
+ 'helloapple': {
+ '1.0': sample_resource
+ }
+ },
+ 'mappings': {
+ 'helloapple': {
+ '0.1.1': sample_mapping
+ }
+ },
+ 'files': sample_files_by_hash
+ }
+ for window in reversed(windows):
+ driver.switch_to.window(window)
+ execute_in_page('initial_data = arguments[0];', initial_data)
+
+ # See if haketilodb.track_*() functions properly return the already-existing
+ # items.
+ execute_in_page(
+ '''
+ function update_item(store_name, change)
+ {
+ console.log('update', ...arguments);
+ const elem_id = `${store_name}_${change.identifier}`;
+ let elem = document.getElementById(elem_id);
+ elem = elem || document.createElement("li");
+ elem.id = elem_id;
+ elem.innerText = JSON.stringify(change.new_val);
+ document.getElementById(store_name).append(elem);
+ if (change.new_val === undefined)
+ elem.remove();
+ }
+
+ let resource_tracking, resource_items, mapping_tracking, mapping_items;
+
+ async function start_tracking()
+ {
+ const update_resource = change => update_item("resources", change);
+ const update_mapping = change => update_item("mappings", change);
+
+ [resource_tracking, resource_items] =
+ await haketilodb.track_resources(update_resource);
+ [mapping_tracking, mapping_items] =
+ await haketilodb.track_mappings(update_mapping);
+
+ for (const item of resource_items)
+ update_resource({identifier: item.identifier, new_val: item});
+ for (const item of mapping_items)
+ update_mapping({identifier: item.identifier, new_val: item});
+ }
+
+ returnval(start_tracking());
+ ''')
+
+ item_counts = driver.execute_script(
+ '''
+ const childcount = id => document.getElementById(id).childElementCount;
+ return ["resources", "mappings"].map(childcount);
+ ''')
+ assert item_counts == [1, 1]
+ resource_json = driver.find_element_by_id('resources_helloapple').text
+ mapping_json = driver.find_element_by_id('mappings_helloapple').text
+ assert json.loads(resource_json) == sample_resource
+ assert json.loads(mapping_json) == sample_mapping
+
+ # See if item additions get tracked properly.
+ driver.switch_to.window(windows[1])
+ sample_resource2 = make_sample_resource()
+ sample_resource2['identifier'] = 'helloapple-copy'
+ sample_mapping2 = make_sample_mapping()
+ sample_mapping2['identifier'] = 'helloapple-copy'
+ sample_data = {
+ 'resources': {
+ 'helloapple-copy': {
+ '1.0': sample_resource2
+ }
+ },
+ 'mappings': {
+ 'helloapple-copy': {
+ '0.1.1': sample_mapping2
+ }
+ },
+ 'files': sample_files_by_hash
+ }
+ execute_in_page('returnval(haketilodb.save_items(arguments[0]));',
+ sample_data)
+
+ driver.switch_to.window(windows[0])
+ driver.implicitly_wait(10)
+ resource_json = driver.find_element_by_id('resources_helloapple-copy').text
+ mapping_json = driver.find_element_by_id('mappings_helloapple-copy').text
+ driver.implicitly_wait(0)
+ assert json.loads(resource_json) == sample_resource2
+ assert json.loads(mapping_json) == sample_mapping2
+
+ # See if item deletions get tracked properly.
+ driver.switch_to.window(windows[1])
+ execute_in_page(
+ '''{
+ async function remove_items()
+ {
+ const store_names = ["resources", "mappings"];
+ const ctx = await haketilodb.start_items_transaction(store_names, {});
+ await haketilodb.remove_resource("helloapple", ctx);
+ await haketilodb.remove_mapping("helloapple-copy", ctx);
+ await haketilodb.finalize_items_transaction(ctx);
+ }
+ returnval(remove_items());
+ }''')
+
+ removed_ids = ['mappings_helloapple-copy', 'resources_helloapple']
+ def condition_items_absent(driver):
+ for id in removed_ids:
+ try:
+ driver.find_element_by_id(id)
+ return False
+ except WebDriverException:
+ pass
+ return True
+
+ driver.switch_to.window(windows[0])
+ WebDriverWait(driver, 10).until(condition_items_absent)