facilitate querying IndexedDB for script files of resource and its dependencies

5 files changed, 377 insertions, 33 deletions

diff --git a/background/indexeddb_files_server.js b/background/indexeddb_files_server.js
new file mode 100644
index 0000000..cf78ca6
--- /dev/null
+++ b/background/indexeddb_files_server.js
@@ -0,0 +1,152 @@
+/**
+ * This file is part of Haketilo.
+ *
+ * Function: Allow content scripts to query IndexedDB through messages to
+ *           background script.
+ *
+ * Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * As additional permission under GNU GPL version 3 section 7, you
+ * may distribute forms of that code without the copy of the GNU
+ * GPL normally required by section 4, provided you include this
+ * license notice and, in case of non-source distribution, a URL
+ * through which recipients can access the Corresponding Source.
+ * If you modify file(s) with this exception, you may extend this
+ * exception to your version of the file(s), but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * As a special exception to the GPL, any HTML file which merely
+ * makes function calls to this code, and for that purpose
+ * includes it by reference shall be deemed a separate work for
+ * copyright law purposes. If you modify this code, you may extend
+ * this exception to your version of the code, but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * I, Wojtek Kosior, thereby promise not to sue for violation of this file's
+ * license. Although I request that you do not make use of this code in a
+ * proprietary program, I am not going to enforce this in court.
+ */
+
+#IMPORT common/indexeddb.js AS haketilodb
+
+#FROM common/browser.js IMPORT browser
+
+async function get_resource_files(getting, id) {
+    if (getting.defs_by_res_id.has(id))
+	return;
+
+    getting.defs_by_res_id.set(id, null);
+
+    const definition = await haketilodb.idb_get(getting.tx, "resource", id);
+    if (!definition)
+	throw {haketilo_error_type: "missing", id};
+
+    getting.defs_by_res_id.set(id, definition);
+
+    const file_proms = (definition.scripts || [])
+	  .map(s => haketilodb.idb_get(getting.tx, "files", s.hash_key));
+
+    const deps_proms = (definition.dependencies || [])
+	  .map(dep_id => get_resource_files(getting, dep_id));
+
+    const files = (await Promise.all(file_proms)).map(f => f.contents);
+    getting.files_by_res_id.set(id, files);
+
+    await Promise.all(deps_proms);
+}
+
+function get_files_list(defs_by_res_id, files_by_res_id, root_id) {
+    const processed = new Set(), to_process = [["start", root_id]],
+	  trace = new Set(), files = [];
+
+    while (to_process.length > 0) {
+	const [what, id] = to_process.pop();
+	if (what === "end") {
+	    trace.delete(id);
+	    files.push(...files_by_res_id.get(id));
+	    continue;
+	}
+
+	if (trace.has(id))
+	    throw {haketilo_error_type: "circular", id};
+
+	if (processed.has(id))
+	    continue;
+
+	trace.add(id);
+	to_process.push(["end", id]);
+	processed.add(id);
+
+	const ds = (defs_by_res_id.get(id).dependencies || []).reverse();
+	ds.forEach(dep_id => to_process.push(["start", dep_id]));
+    }
+
+    return files;
+}
+
+async function send_resource_files(root_resource_id, send_cb) {
+    const db = await haketilodb.get();
+    const getting = {
+	defs_by_res_id:  new Map(),
+	files_by_res_id: new Map(),
+	tx:              db.transaction(["files", "resource"])
+    };
+
+    let prom_cbs, prom = new Promise((...cbs) => prom_cbs = cbs);
+
+    getting.tx.onerror = e => prom_cbs[1]({haketilo_error_type: "db", e});
+
+    get_resource_files(getting, root_resource_id, new Set()).then(...prom_cbs);
+
+    try {
+	await prom;
+	const files = get_files_list(
+	    getting.defs_by_res_id,
+	    getting.files_by_res_id,
+	    root_resource_id
+	);
+	var to_send = {files};
+    } catch(e) {
+	if (typeof e === "object" && "haketilo_error_type" in e) {
+	    if (e.haketilo_error_type === "db") {
+		console.error(e.e);
+		delete e.e;
+	    }
+	    var to_send = {error: e};
+	} else {
+	    console.error(e);
+	    var to_send = {error: {haketilo_error_type: "other"}};
+	}
+    }
+
+    send_cb(to_send);
+}
+
+function on_indexeddb_files_request([type, resource_id], sender, respond_cb) {
+    if (type !== "indexeddb_files")
+	return;
+
+    send_resource_files(resource_id, respond_cb);
+
+    return true;
+}
+
+function start() {
+    browser.runtime.onMessage.addListener(on_indexeddb_files_request);
+}
+#EXPORT start
diff --git a/common/message_server.js b/common/message_server.js
index 2b93ed6..80cefd5 100644
--- a/common/message_server.js
+++ b/common/message_server.js
@@ -106,5 +106,4 @@ function connect_to_background(magic)
     listeners[magic](ports[0]);
     return ports[1];
 }
-
 #EXPORT connect_to_background
diff --git a/test/unit/test_indexeddb.py b/test/unit/test_indexeddb.py
index aea633b..b320cff 100644
--- a/test/unit/test_indexeddb.py
+++ b/test/unit/test_indexeddb.py
@@ -51,16 +51,6 @@ def make_sample_mapping():
         'identifier': 'helloapple'
     }
 
-def mock_broadcast(execute_in_page):
-    execute_in_page(
-        '''{
-        const broadcast_mock = {};
-        const nop = () => {};
-        for (const key in broadcast)
-            broadcast_mock[key] = nop;
-        broadcast = broadcast_mock;
-        }''')
-
 @pytest.mark.get_page('https://gotmyowndoma.in')
 def test_haketilodb_item_modifications(driver, execute_in_page):
     """
diff --git a/test/unit/test_indexeddb_files_server.py b/test/unit/test_indexeddb_files_server.py
new file mode 100644
index 0000000..ab69d9d
--- /dev/null
+++ b/test/unit/test_indexeddb_files_server.py
@@ -0,0 +1,169 @@
+# SPDX-License-Identifier: CC0-1.0
+
+"""
+Haketilo unit tests - serving indexeddb resource script files to content scripts
+"""
+
+# This file is part of Haketilo
+#
+# Copyright (C) 2021,2022 Wojtek Kosior <koszko@koszko.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the CC0 1.0 Universal License as published by
+# the Creative Commons Corporation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# CC0 1.0 Universal License for more details.
+
+import pytest
+import copy
+from uuid import uuid4
+from selenium.webdriver.support.ui import WebDriverWait
+
+from ..script_loader import load_script
+from .utils import *
+
+"""
+How many test resources we're going to have.
+"""
+count = 15
+
+sample_files_list = [(f'file_{n}_{i}', f'contents {n} {i}')
+                       for n in range(count) for i in range(2)]
+
+sample_files = dict(sample_files_list)
+
+sample_files, sample_files_by_hash = make_sample_files(sample_files)
+
+def make_sample_resource_with_deps(n):
+    resource = make_sample_resource(with_files=False)
+
+    resource['identifier'] = f'res-{n}'
+    resource['dependencies'] = [f'res-{m}'
+                                for m in range(max(n - 4, 0), n)]
+    resource['scripts'] = [sample_file_ref(f'file_{n}_{i}', sample_files)
+                           for i in range(2)]
+
+    return resource
+
+resources = [make_sample_resource_with_deps(n) for n in range(count)]
+
+sample_data = {
+    'resources': sample_data_dict(resources),
+    'mapping': {},
+    'files': sample_files_by_hash
+}
+
+def prepare_test_page(initial_indexeddb_data, execute_in_page):
+    js = load_script('background/indexeddb_files_server.js',
+                     code_to_add='#IMPORT common/broadcast.js')
+    execute_in_page(js)
+
+    mock_broadcast(execute_in_page)
+    clear_indexeddb(execute_in_page)
+
+    execute_in_page(
+        '''
+        let registered_listener;
+        const new_addListener = cb => registered_listener = cb;
+
+        browser = {runtime: {onMessage: {addListener: new_addListener}}};
+
+        haketilodb.save_items(arguments[0]);
+
+        start();
+        ''',
+        initial_indexeddb_data)
+
+@pytest.mark.get_page('https://gotmyowndoma.in')
+def test_indexeddb_files_server_normal_usage(driver, execute_in_page):
+    """
+    Test querying resource files (with resource dependency resolution)
+    from IndexedDB and serving them in messages to content scripts.
+    """
+    prepare_test_page(sample_data, execute_in_page)
+
+    # Verify other types of messages are ignored.
+    function_returned_value = execute_in_page(
+        '''
+        returnval(registered_listener(["???"], {},
+                  () => location.reload()));
+        ''')
+    assert function_returned_value == None
+
+    # Verify single resource's files get properly resolved.
+    function_returned_value = execute_in_page(
+        '''
+        var result_cb, contents_prom = new Promise(cb => result_cb = cb);
+
+        returnval(registered_listener(["indexeddb_files", "res-0"],
+                                      {}, result_cb));
+        ''')
+    assert function_returned_value == True
+
+    assert execute_in_page('returnval(contents_prom);') == \
+        {'files': [tuple[1] for tuple in sample_files_list[0:2]]}
+
+    # Verify multiple resources' files get properly resolved.
+    function_returned_value = execute_in_page(
+        '''
+        var result_cb, contents_prom = new Promise(cb => result_cb = cb);
+
+        returnval(registered_listener(["indexeddb_files", arguments[0]],
+                                      {}, result_cb));
+        ''',
+        f'res-{count - 1}')
+    assert function_returned_value == True
+
+    assert execute_in_page('returnval(contents_prom);') == \
+        {'files': [tuple[1] for tuple in sample_files_list]}
+
+@pytest.mark.get_page('https://gotmyowndoma.in')
+@pytest.mark.parametrize('error', [
+    'missing',
+    'circular',
+    'db',
+    'other'
+])
+def test_indexeddb_files_server_errors(driver, execute_in_page, error):
+    """
+    Test reporting of errors when querying resource files (with resource
+    dependency resolution) from IndexedDB and serving them in messages to
+    content scripts.
+    """
+    sample_data_copy = copy.deepcopy(sample_data)
+
+    if error == 'missing':
+        del sample_data_copy['resources']['res-3']
+    elif error == 'circular':
+        res3_defs = sample_data_copy['resources']['res-3'].values()
+        next(iter(res3_defs))['dependencies'].append('res-8')
+
+    prepare_test_page(sample_data_copy, execute_in_page)
+
+    if error == 'db':
+        execute_in_page('haketilodb.idb_get = t => t.onerror("oooops");')
+    elif error == 'other':
+        execute_in_page('haketilodb.idb_get = () => {throw "oooops"};')
+
+    response = execute_in_page(
+        '''
+        var result_cb, contents_prom = new Promise(cb => result_cb = cb);
+
+        registered_listener(["indexeddb_files", arguments[0]],
+                            {}, result_cb);
+
+        returnval(contents_prom);
+        ''',
+        f'res-{count - 1}')
+
+    assert response['error']['haketilo_error_type'] == error
+
+    if error == 'missing':
+        assert response['error']['id'] == 'res-3'
+    elif error == 'circular':
+        assert response['error']['id'] in ('res-3', 'res-8')
+    elif error not in ('db', 'other'):
+        raise Exception('made a typo in test function params?')
diff --git a/test/unit/utils.py b/test/unit/utils.py
index 6f0236d..56880d5 100644
--- a/test/unit/utils.py
+++ b/test/unit/utils.py
@@ -42,29 +42,51 @@ def sample_file(contents):
         'contents': contents
     }
 
-sample_files = {
-    'report.spdx':              sample_file('<!-- dummy report -->'),
-    'LICENSES/somelicense.txt': sample_file('Permission is granted...'),
-    'LICENSES/CC0-1.0.txt':     sample_file('Dummy Commons...'),
-    'hello.js':                 sample_file('console.log("uńićódę hello!");\n'),
-    'bye.js':                   sample_file('console.log("bye!");\n'),
-    'combined.js':              sample_file('console.log("hello!\\nbye!");\n'),
-    'README.md':                sample_file('# Python Frobnicator\n...')
-}
-
-sample_files_by_hash = dict([[file['hash_key'], file['contents']]
-                             for file in sample_files.values()])
-
-def sample_file_ref(file_name):
-    return {'file': file_name, 'hash_key': sample_files[file_name]['hash_key']}
-
-def make_sample_mapping():
+def make_sample_files(names_contents):
+    """
+    Take a dict mapping file names to file contents. Return, as a tuple, dicts
+    mapping file names to file objects (dicts) and file hash keys to file
+    contents.
+    """
+    sample_files = dict([(name, sample_file(contents))
+                         for name, contents in names_contents.items()])
+
+    sample_files_by_hash = dict([[file['hash_key'], file['contents']]
+                                 for file in sample_files.values()])
+
+    return sample_files, sample_files_by_hash
+
+sample_files, sample_files_by_hash = make_sample_files({
+    'report.spdx':              '<!-- dummy report -->',
+    'LICENSES/somelicense.txt': 'Permission is granted...',
+    'LICENSES/CC0-1.0.txt':     'Dummy Commons...',
+    'hello.js':                 'console.log("uńićódę hello!");\n',
+    'bye.js':                   'console.log("bye!");\n',
+    'combined.js':              'console.log("hello!\\nbye!");\n',
+    'README.md':                '# Python Frobnicator\n...'
+})
+
+def sample_file_ref(file_name, sample_files_dict=sample_files):
+    """
+    Return a dictionary suitable for using as file reference in resource/mapping
+    definition.
+    """
+    return {
+        'file': file_name,
+        'hash_key': sample_files_dict[file_name]['hash_key']
+    }
+
+def make_sample_mapping(with_files=True):
+    """
+    Procude a sample mapping definition that can be dumped to JSON and put into
+    Haketilo's IndexedDB.
+    """
     return {
         'source_name': 'example-org-fixes-new',
         'source_copyright': [
             sample_file_ref('report.spdx'),
             sample_file_ref('LICENSES/CC0-1.0.txt')
-        ],
+        ] if with_files else [],
         'type': 'mapping',
         'identifier': 'example-org-minimal',
         'long_name': 'Example.org Minimal',
@@ -81,13 +103,17 @@ def make_sample_mapping():
         }
     }
 
-def make_sample_resource():
+def make_sample_resource(with_files=True):
+    """
+    Procude a sample resource definition that can be dumped to JSON and put into
+    Haketilo's IndexedDB.
+    """
     return {
         'source_name': 'hello',
         'source_copyright': [
             sample_file_ref('report.spdx'),
             sample_file_ref('LICENSES/CC0-1.0.txt')
-        ],
+        ] if with_files else [],
         'type': 'resource',
         'identifier': 'helloapple',
         'long_name': 'Hello Apple',
@@ -99,7 +125,7 @@ def make_sample_resource():
         'scripts': [
             sample_file_ref('hello.js'),
             sample_file_ref('bye.js')
-        ]
+        ] if with_files else []
     }
 
 def item_version_string(definition, include_revision=False):
@@ -113,7 +139,7 @@ def item_version_string(definition, include_revision=False):
 
 def sample_data_dict(items):
     """
-    Some indexeddb functions expect saved items to be provided in a nested dict
+    Some IndexedDB functions expect saved items to be provided in a nested dict
     that makes them queryable by identifier by version. This function converts
     items list to such dict.
     """
@@ -202,6 +228,14 @@ def are_scripts_allowed(driver, nonce=None):
             ''',
             nonce)
 
+def mock_broadcast(execute_in_page):
+    """
+    Make all broadcast operations no-ops (broadcast must be imported).
+    """
+    execute_in_page(
+        'Object.keys(broadcast).forEach(k => broadcast[k] = () => {});'
+    )
+
 def mock_cacher(execute_in_page):
     """
     Some parts of code depend on content/repo_query_cacher.js and