From 5c58b3d65e370ebd3dadc1133157c73c6afc84af Mon Sep 17 00:00:00 2001 From: Wojtek Kosior Date: Thu, 27 Jan 2022 19:35:44 +0100 Subject: facilitate querying IndexedDB for script files of resource and its dependencies --- background/indexeddb_files_server.js | 152 +++++++++++++++++++++++++++ common/message_server.js | 1 - test/unit/test_indexeddb.py | 10 -- test/unit/test_indexeddb_files_server.py | 169 +++++++++++++++++++++++++++++++ test/unit/utils.py | 78 ++++++++++---- 5 files changed, 377 insertions(+), 33 deletions(-) create mode 100644 background/indexeddb_files_server.js create mode 100644 test/unit/test_indexeddb_files_server.py diff --git a/background/indexeddb_files_server.js b/background/indexeddb_files_server.js new file mode 100644 index 0000000..cf78ca6 --- /dev/null +++ b/background/indexeddb_files_server.js @@ -0,0 +1,152 @@ +/** + * This file is part of Haketilo. + * + * Function: Allow content scripts to query IndexedDB through messages to + * background script. + * + * Copyright (C) 2022 Wojtek Kosior + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * As additional permission under GNU GPL version 3 section 7, you + * may distribute forms of that code without the copy of the GNU + * GPL normally required by section 4, provided you include this + * license notice and, in case of non-source distribution, a URL + * through which recipients can access the Corresponding Source. + * If you modify file(s) with this exception, you may extend this + * exception to your version of the file(s), but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * As a special exception to the GPL, any HTML file which merely + * makes function calls to this code, and for that purpose + * includes it by reference shall be deemed a separate work for + * copyright law purposes. If you modify this code, you may extend + * this exception to your version of the code, but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * I, Wojtek Kosior, thereby promise not to sue for violation of this file's + * license. Although I request that you do not make use of this code in a + * proprietary program, I am not going to enforce this in court. + */ + +#IMPORT common/indexeddb.js AS haketilodb + +#FROM common/browser.js IMPORT browser + +async function get_resource_files(getting, id) { + if (getting.defs_by_res_id.has(id)) + return; + + getting.defs_by_res_id.set(id, null); + + const definition = await haketilodb.idb_get(getting.tx, "resource", id); + if (!definition) + throw {haketilo_error_type: "missing", id}; + + getting.defs_by_res_id.set(id, definition); + + const file_proms = (definition.scripts || []) + .map(s => haketilodb.idb_get(getting.tx, "files", s.hash_key)); + + const deps_proms = (definition.dependencies || []) + .map(dep_id => get_resource_files(getting, dep_id)); + + const files = (await Promise.all(file_proms)).map(f => f.contents); + getting.files_by_res_id.set(id, files); + + await Promise.all(deps_proms); +} + +function get_files_list(defs_by_res_id, files_by_res_id, root_id) { + const processed = new Set(), to_process = [["start", root_id]], + trace = new Set(), files = []; + + while (to_process.length > 0) { + const [what, id] = to_process.pop(); + if (what === "end") { + trace.delete(id); + files.push(...files_by_res_id.get(id)); + continue; + } + + if (trace.has(id)) + throw {haketilo_error_type: "circular", id}; + + if (processed.has(id)) + continue; + + trace.add(id); + to_process.push(["end", id]); + processed.add(id); + + const ds = (defs_by_res_id.get(id).dependencies || []).reverse(); + ds.forEach(dep_id => to_process.push(["start", dep_id])); + } + + return files; +} + +async function send_resource_files(root_resource_id, send_cb) { + const db = await haketilodb.get(); + const getting = { + defs_by_res_id: new Map(), + files_by_res_id: new Map(), + tx: db.transaction(["files", "resource"]) + }; + + let prom_cbs, prom = new Promise((...cbs) => prom_cbs = cbs); + + getting.tx.onerror = e => prom_cbs[1]({haketilo_error_type: "db", e}); + + get_resource_files(getting, root_resource_id, new Set()).then(...prom_cbs); + + try { + await prom; + const files = get_files_list( + getting.defs_by_res_id, + getting.files_by_res_id, + root_resource_id + ); + var to_send = {files}; + } catch(e) { + if (typeof e === "object" && "haketilo_error_type" in e) { + if (e.haketilo_error_type === "db") { + console.error(e.e); + delete e.e; + } + var to_send = {error: e}; + } else { + console.error(e); + var to_send = {error: {haketilo_error_type: "other"}}; + } + } + + send_cb(to_send); +} + +function on_indexeddb_files_request([type, resource_id], sender, respond_cb) { + if (type !== "indexeddb_files") + return; + + send_resource_files(resource_id, respond_cb); + + return true; +} + +function start() { + browser.runtime.onMessage.addListener(on_indexeddb_files_request); +} +#EXPORT start diff --git a/common/message_server.js b/common/message_server.js index 2b93ed6..80cefd5 100644 --- a/common/message_server.js +++ b/common/message_server.js @@ -106,5 +106,4 @@ function connect_to_background(magic) listeners[magic](ports[0]); return ports[1]; } - #EXPORT connect_to_background diff --git a/test/unit/test_indexeddb.py b/test/unit/test_indexeddb.py index aea633b..b320cff 100644 --- a/test/unit/test_indexeddb.py +++ b/test/unit/test_indexeddb.py @@ -51,16 +51,6 @@ def make_sample_mapping(): 'identifier': 'helloapple' } -def mock_broadcast(execute_in_page): - execute_in_page( - '''{ - const broadcast_mock = {}; - const nop = () => {}; - for (const key in broadcast) - broadcast_mock[key] = nop; - broadcast = broadcast_mock; - }''') - @pytest.mark.get_page('https://gotmyowndoma.in') def test_haketilodb_item_modifications(driver, execute_in_page): """ diff --git a/test/unit/test_indexeddb_files_server.py b/test/unit/test_indexeddb_files_server.py new file mode 100644 index 0000000..ab69d9d --- /dev/null +++ b/test/unit/test_indexeddb_files_server.py @@ -0,0 +1,169 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - serving indexeddb resource script files to content scripts +""" + +# This file is part of Haketilo +# +# Copyright (C) 2021,2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +import copy +from uuid import uuid4 +from selenium.webdriver.support.ui import WebDriverWait + +from ..script_loader import load_script +from .utils import * + +""" +How many test resources we're going to have. +""" +count = 15 + +sample_files_list = [(f'file_{n}_{i}', f'contents {n} {i}') + for n in range(count) for i in range(2)] + +sample_files = dict(sample_files_list) + +sample_files, sample_files_by_hash = make_sample_files(sample_files) + +def make_sample_resource_with_deps(n): + resource = make_sample_resource(with_files=False) + + resource['identifier'] = f'res-{n}' + resource['dependencies'] = [f'res-{m}' + for m in range(max(n - 4, 0), n)] + resource['scripts'] = [sample_file_ref(f'file_{n}_{i}', sample_files) + for i in range(2)] + + return resource + +resources = [make_sample_resource_with_deps(n) for n in range(count)] + +sample_data = { + 'resources': sample_data_dict(resources), + 'mapping': {}, + 'files': sample_files_by_hash +} + +def prepare_test_page(initial_indexeddb_data, execute_in_page): + js = load_script('background/indexeddb_files_server.js', + code_to_add='#IMPORT common/broadcast.js') + execute_in_page(js) + + mock_broadcast(execute_in_page) + clear_indexeddb(execute_in_page) + + execute_in_page( + ''' + let registered_listener; + const new_addListener = cb => registered_listener = cb; + + browser = {runtime: {onMessage: {addListener: new_addListener}}}; + + haketilodb.save_items(arguments[0]); + + start(); + ''', + initial_indexeddb_data) + +@pytest.mark.get_page('https://gotmyowndoma.in') +def test_indexeddb_files_server_normal_usage(driver, execute_in_page): + """ + Test querying resource files (with resource dependency resolution) + from IndexedDB and serving them in messages to content scripts. + """ + prepare_test_page(sample_data, execute_in_page) + + # Verify other types of messages are ignored. + function_returned_value = execute_in_page( + ''' + returnval(registered_listener(["???"], {}, + () => location.reload())); + ''') + assert function_returned_value == None + + # Verify single resource's files get properly resolved. + function_returned_value = execute_in_page( + ''' + var result_cb, contents_prom = new Promise(cb => result_cb = cb); + + returnval(registered_listener(["indexeddb_files", "res-0"], + {}, result_cb)); + ''') + assert function_returned_value == True + + assert execute_in_page('returnval(contents_prom);') == \ + {'files': [tuple[1] for tuple in sample_files_list[0:2]]} + + # Verify multiple resources' files get properly resolved. + function_returned_value = execute_in_page( + ''' + var result_cb, contents_prom = new Promise(cb => result_cb = cb); + + returnval(registered_listener(["indexeddb_files", arguments[0]], + {}, result_cb)); + ''', + f'res-{count - 1}') + assert function_returned_value == True + + assert execute_in_page('returnval(contents_prom);') == \ + {'files': [tuple[1] for tuple in sample_files_list]} + +@pytest.mark.get_page('https://gotmyowndoma.in') +@pytest.mark.parametrize('error', [ + 'missing', + 'circular', + 'db', + 'other' +]) +def test_indexeddb_files_server_errors(driver, execute_in_page, error): + """ + Test reporting of errors when querying resource files (with resource + dependency resolution) from IndexedDB and serving them in messages to + content scripts. + """ + sample_data_copy = copy.deepcopy(sample_data) + + if error == 'missing': + del sample_data_copy['resources']['res-3'] + elif error == 'circular': + res3_defs = sample_data_copy['resources']['res-3'].values() + next(iter(res3_defs))['dependencies'].append('res-8') + + prepare_test_page(sample_data_copy, execute_in_page) + + if error == 'db': + execute_in_page('haketilodb.idb_get = t => t.onerror("oooops");') + elif error == 'other': + execute_in_page('haketilodb.idb_get = () => {throw "oooops"};') + + response = execute_in_page( + ''' + var result_cb, contents_prom = new Promise(cb => result_cb = cb); + + registered_listener(["indexeddb_files", arguments[0]], + {}, result_cb); + + returnval(contents_prom); + ''', + f'res-{count - 1}') + + assert response['error']['haketilo_error_type'] == error + + if error == 'missing': + assert response['error']['id'] == 'res-3' + elif error == 'circular': + assert response['error']['id'] in ('res-3', 'res-8') + elif error not in ('db', 'other'): + raise Exception('made a typo in test function params?') diff --git a/test/unit/utils.py b/test/unit/utils.py index 6f0236d..56880d5 100644 --- a/test/unit/utils.py +++ b/test/unit/utils.py @@ -42,29 +42,51 @@ def sample_file(contents): 'contents': contents } -sample_files = { - 'report.spdx': sample_file(''), - 'LICENSES/somelicense.txt': sample_file('Permission is granted...'), - 'LICENSES/CC0-1.0.txt': sample_file('Dummy Commons...'), - 'hello.js': sample_file('console.log("uńićódę hello!");\n'), - 'bye.js': sample_file('console.log("bye!");\n'), - 'combined.js': sample_file('console.log("hello!\\nbye!");\n'), - 'README.md': sample_file('# Python Frobnicator\n...') -} - -sample_files_by_hash = dict([[file['hash_key'], file['contents']] - for file in sample_files.values()]) - -def sample_file_ref(file_name): - return {'file': file_name, 'hash_key': sample_files[file_name]['hash_key']} - -def make_sample_mapping(): +def make_sample_files(names_contents): + """ + Take a dict mapping file names to file contents. Return, as a tuple, dicts + mapping file names to file objects (dicts) and file hash keys to file + contents. + """ + sample_files = dict([(name, sample_file(contents)) + for name, contents in names_contents.items()]) + + sample_files_by_hash = dict([[file['hash_key'], file['contents']] + for file in sample_files.values()]) + + return sample_files, sample_files_by_hash + +sample_files, sample_files_by_hash = make_sample_files({ + 'report.spdx': '', + 'LICENSES/somelicense.txt': 'Permission is granted...', + 'LICENSES/CC0-1.0.txt': 'Dummy Commons...', + 'hello.js': 'console.log("uńićódę hello!");\n', + 'bye.js': 'console.log("bye!");\n', + 'combined.js': 'console.log("hello!\\nbye!");\n', + 'README.md': '# Python Frobnicator\n...' +}) + +def sample_file_ref(file_name, sample_files_dict=sample_files): + """ + Return a dictionary suitable for using as file reference in resource/mapping + definition. + """ + return { + 'file': file_name, + 'hash_key': sample_files_dict[file_name]['hash_key'] + } + +def make_sample_mapping(with_files=True): + """ + Procude a sample mapping definition that can be dumped to JSON and put into + Haketilo's IndexedDB. + """ return { 'source_name': 'example-org-fixes-new', 'source_copyright': [ sample_file_ref('report.spdx'), sample_file_ref('LICENSES/CC0-1.0.txt') - ], + ] if with_files else [], 'type': 'mapping', 'identifier': 'example-org-minimal', 'long_name': 'Example.org Minimal', @@ -81,13 +103,17 @@ def make_sample_mapping(): } } -def make_sample_resource(): +def make_sample_resource(with_files=True): + """ + Procude a sample resource definition that can be dumped to JSON and put into + Haketilo's IndexedDB. + """ return { 'source_name': 'hello', 'source_copyright': [ sample_file_ref('report.spdx'), sample_file_ref('LICENSES/CC0-1.0.txt') - ], + ] if with_files else [], 'type': 'resource', 'identifier': 'helloapple', 'long_name': 'Hello Apple', @@ -99,7 +125,7 @@ def make_sample_resource(): 'scripts': [ sample_file_ref('hello.js'), sample_file_ref('bye.js') - ] + ] if with_files else [] } def item_version_string(definition, include_revision=False): @@ -113,7 +139,7 @@ def item_version_string(definition, include_revision=False): def sample_data_dict(items): """ - Some indexeddb functions expect saved items to be provided in a nested dict + Some IndexedDB functions expect saved items to be provided in a nested dict that makes them queryable by identifier by version. This function converts items list to such dict. """ @@ -202,6 +228,14 @@ def are_scripts_allowed(driver, nonce=None): ''', nonce) +def mock_broadcast(execute_in_page): + """ + Make all broadcast operations no-ops (broadcast must be imported). + """ + execute_in_page( + 'Object.keys(broadcast).forEach(k => broadcast[k] = () => {});' + ) + def mock_cacher(execute_in_page): """ Some parts of code depend on content/repo_query_cacher.js and -- cgit v1.2.3