diff options
| -rw-r--r-- | Makefile.in | 2 | ||||
| -rw-r--r-- | background/main.js | 13 | ||||
| -rwxr-xr-x | build.sh | 11 | ||||
| -rw-r--r-- | common/entities.js | 148 | ||||
| -rw-r--r-- | common/indexeddb.js | 270 | ||||
| -rw-r--r-- | compute_scripts.awk | 7 | ||||
| -rw-r--r-- | test/script_loader.py | 7 | ||||
| -rw-r--r-- | test/unit/test_indexeddb.py | 134 | ||||
| -rwxr-xr-x | write_exports_init.sh | 29 |
9 files changed, 598 insertions, 23 deletions
diff --git a/Makefile.in b/Makefile.in index 5a376e1..5291299 100644 --- a/Makefile.in +++ b/Makefile.in @@ -79,7 +79,7 @@ test-environment: test/certs/rootCA.pem test/certs/site.key clean mostlyclean: rm -rf mozilla-unpacked chromium-unpacked haketilo-$(version) rm -f mozilla-build.zip chromium-build.zip haketilo-$(version).tar.gz \ - haketilo-$(version).tar + haketilo-$(version).tar exports_init.js rm -rf test/certs rm -rf $$(find . -name geckodriver.log) rm -rf $$(find . -type d -name __pycache__) diff --git a/background/main.js b/background/main.js index a4d3f0e..2809334 100644 --- a/background/main.js +++ b/background/main.js @@ -3,7 +3,8 @@ * * Function: Main background script. * - * Copyright (C) 2021 Wojtek Kosior + * Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org> + * Copyright (C) 2021 Jahoti <jahoti@envs.net> * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -43,6 +44,7 @@ /* * IMPORTS_START + * IMPORT initial_data * IMPORT TYPE_PREFIX * IMPORT get_storage * IMPORT light_storage @@ -70,13 +72,8 @@ async function init_ext(install_details) await storage.clear(); - /* - * Below we add sample settings to the extension. - */ - - for (let setting of // The next line is replaced with the contents of /default_settings.json by the build script - `DEFAULT SETTINGS` - ) { + /* Below we add sample settings to the extension. */ + for (let setting of initial_data) { let [key, value] = Object.entries(setting)[0]; storage.set(key[0], key.substring(1), value); } @@ -123,16 +123,7 @@ s^_CONTENTSCRIPTS_^$CONTENTSCRIPTS^" \ fi done - # A hack to insert the contents of default_settings.json at the appropriate - # location in background/main.js. Uses an internal sed expression to escape - # and indent the JSON file for use in the external sed expression. - sed -i 's/^ `DEFAULT SETTINGS`$/'"$(sed -E 's/([\\\&\/])/\\\1/g; s/^/ /; s/$/\\/' < default_settings.json) "/g "$BUILDDIR"/background/main.js - - if [ "$BROWSER" = "chromium" ]; then - cp CHROMIUM_exports_init.js "$BUILDDIR"/exports_init.js - else - cp MOZILLA_exports_init.js "$BUILDDIR"/exports_init.js - fi + ./write_exports_init.sh "$BROWSER" "$BUILDDIR" default_settings.json cp -r copyright licenses/ "$BUILDDIR" cp dummy "$BUILDDIR" diff --git a/common/entities.js b/common/entities.js new file mode 100644 index 0000000..46836b5 --- /dev/null +++ b/common/entities.js @@ -0,0 +1,148 @@ +/** + * This file is part of Haketilo. + * + * Function: Operations on resources and mappings. + * + * Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * As additional permission under GNU GPL version 3 section 7, you + * may distribute forms of that code without the copy of the GNU + * GPL normally required by section 4, provided you include this + * license notice and, in case of non-source distribution, a URL + * through which recipients can access the Corresponding Source. + * If you modify file(s) with this exception, you may extend this + * exception to your version of the file(s), but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * As a special exception to the GPL, any HTML file which merely + * makes function calls to this code, and for that purpose + * includes it by reference shall be deemed a separate work for + * copyright law purposes. If you modify this code, you may extend + * this exception to your version of the code, but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + * + * I, Wojtek Kosior, thereby promise not to sue for violation of this file's + * license. Although I request that you do not make use this code in a + * proprietary program, I am not going to enforce this in court. + */ + +/* + * Convert ver_str into an array representation, e.g. for ver_str="4.6.13.0" + * return [4, 6, 13, 0]. + */ +const parse_version = ver_str => ver_str.split(".").map(parseInt); + +/* + * ver is an array of integers. rev is an optional integer. Produce string + * representation of version (optionally with revision number), like: + * 1.2.3-5 + * No version normalization is performed. + */ +const version_string = (ver, rev=0) => ver.join(".") + (rev ? `-${rev}` : ""); + +/* vers should be an array of comparable values. Return the greatest one. */ +const max = vals => Array.reduce(vals, (v1, v2) => v1 > v2 ? v1 : v2); + +/* + * versioned_item should be a dict with keys being version strings and values + * being definitions of the respective versions of a single resource/mapping. + * Example: + * { + * "1": { + * version: [1]//, + * // more stuff + * }, + * "1.1": { + * version: [1, 1]//, + * // more stuff + * } + * } + * + * Returns the definition with the highest version. + */ +function get_newest_version(versioned_item) +{ + const best_ver = max(Object.keys(versioned_item).map(parse_version)); + return versioned_item[version_string(best_ver)]; +} + +/* + * item is a definition of a resource or mapping. Yield all file references + * (objects with `file` and `sha256` properties) this definition has. + */ +function* get_used_files(item) +{ + for (const file of item.source_copyright) + yield file; + + if (item.type === "resource") { + for (const file of item.scripts || []) + yield file; + } +} + +const entities = { + get_newest: get_newest_version, + get_files: get_used_files +}; + +/* + * EXPORTS_START + * EXPORT entities + * EXPORTS_END + */ + +/* + * Note: the functions below were overeagerly written and are not used now but + * might prove useful to once we add more functionalities and are hence kept... + */ + +/* + * Clone recursively all objects. Leave other items (arrays, strings) untouched. + */ +function deep_object_copy(object) +{ + const orig = {object}; + const result = {}; + const to_copy = [[orig, {}]]; + + while (to_copy.length > 0) { + const [object, copy] = to_copy.pop(); + + for (const [key, value] of Object.entries(object)) { + copy[key] = value; + + if (typeof value === "object" && !Array.isArray(value)) { + const value_copy = {}; + to_copy.push([value, value_copy]); + copy[key] = value_copy; + } + } + } + + return result.orig; +} + +/* helper function for normalize_version() */ +const version_reductor = (acc, n) => [...(n || acc.length ? [n] : []), ...acc]; +/* + * ver is an array of integers. Strip right-most zeroes from ver. + * + * Returns a *new* array. Doesn't modify its argument. + */ +const normalize_version = ver => Array.reduceRight(ver, version_reductor, []); diff --git a/common/indexeddb.js b/common/indexeddb.js new file mode 100644 index 0000000..d0bacc4 --- /dev/null +++ b/common/indexeddb.js @@ -0,0 +1,270 @@ +/** + * This file is part of Haketilo. + * + * Function: Facilitate use of IndexedDB within Haketilo. + * + * Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * As additional permission under GNU GPL version 3 section 7, you + * may distribute forms of that code without the copy of the GNU + * GPL normally required by section 4, provided you include this + * license notice and, in case of non-source distribution, a URL + * through which recipients can access the Corresponding Source. + * If you modify file(s) with this exception, you may extend this + * exception to your version of the file(s), but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * As a special exception to the GPL, any HTML file which merely + * makes function calls to this code, and for that purpose + * includes it by reference shall be deemed a separate work for + * copyright law purposes. If you modify this code, you may extend + * this exception to your version of the code, but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + * + * I, Wojtek Kosior, thereby promise not to sue for violation of this file's + * license. Although I request that you do not make use this code in a + * proprietary program, I am not going to enforce this in court. + */ + +/* + * IMPORTS_START + * IMPORT initial_data + * IMPORT entities + * IMPORTS_END + */ + +/* Update when changes are made to database schema. Must have 3 elements */ +const db_version = [1, 0, 0]; + +const nr_reductor = ([i, s], num) => [i - 1, s + num * 1024 ** i]; +const version_nr = ver => Array.reduce(ver.slice(0, 3), nr_reductor, [2, 0])[1]; + +const stores = [ + ["files", {keyPath: "sha256"}], + ["file_uses", {keyPath: "sha256"}], + ["resources", {keyPath: "identifier"}], + ["mappings", {keyPath: "identifier"}] +]; + +let db = null; + +/* Generate a Promise that resolves when an IndexedDB request succeeds. */ +async function wait_request(idb_request) +{ + let resolve, reject; + const waiter = new Promise((...cbs) => [resolve, reject] = cbs); + [idb_request.onsuccess, idb_request.onerror] = [resolve, reject]; + return waiter; +} + +/* asynchronous wrapper for IDBObjectStore's get() method. */ +async function idb_get(transaction, store_name, key) +{ + const req = transaction.objectStore(store_name).get(key); + return (await wait_request(req)).target.result; +} + +/* asynchronous wrapper for IDBObjectStore's put() method. */ +async function idb_put(transaction, store_name, object) +{ + return wait_request(transaction.objectStore(store_name).put(object)); +} + +/* asynchronous wrapper for IDBObjectStore's delete() method. */ +async function idb_del(transaction, store_name, key) +{ + return wait_request(transaction.objectStore(store_name).delete(key)); +} + +/* Open haketilo database, asynchronously return an IDBDatabase object. */ +async function get_db(initialization_data=initial_data) +{ + if (db) + return db; + + let resolve, reject; + const waiter = new Promise((...cbs) => [resolve, reject] = cbs); + + const request = indexedDB.open("haketilo", version_nr(db_version)); + request.onsuccess = resolve; + request.onerror = ev => reject("db error: " + ev.target.errorCode); + request.onupgradeneeded = resolve; + + const event = await waiter; + const opened_db = event.target.result; + + if (event instanceof IDBVersionChangeEvent) { + /* + * When we move to a new database schema, we will add upgrade logic + * here. + */ + if (event.oldVersion > 0) + throw "bad db version: " + event.oldVersion; + + let store; + for (const [store_name, key_mode] of stores) + store = opened_db.createObjectStore(store_name, key_mode); + + await new Promise(resolve => store.transaction.oncomplete = resolve); + + save_items(db, initialization_data); + } + + db = opened_db; + + return db; +} + +/* + * How a sample data argument to the function below might look like: + * + * data = { + * resources: { + * "resource1": { + * "1": { + * // some stuff + * }, + * "1.1": { + * // some stuff + * } + * }, + * "resource2": { + * "0.4.3": { + * // some stuff + * } + * }, + * }, + * mappings: { + * "mapping1": { + * "2": { + * // some stuff + * } + * }, + * "mapping2": { + * "0.1": { + * // some stuff + * } + * }, + * }, + * files: { + * "sha256-f9444510dc7403e41049deb133f6892aa6a63c05591b2b59e4ee5b234d7bbd99": "console.log(\"hello\");\n", + * "sha256-b857cd521cc82fff30f0d316deba38b980d66db29a5388eb6004579cf743c6fd": "console.log(\"bye\");" + * } + * } + */ +async function save_items(db, data) +{ + const files = data.files; + const resources = + Object.values(data.resources || []).map(entities.get_newest); + const mappings = + Object.values(data.mappings || []).map(entities.get_newest); + + resources.concat(mappings).forEach(i => save_item(i, data.files, db)); +} + +/* helper function of save_item() */ +async function get_file_uses(transaction, file_uses_sha256, file_ref) +{ + let uses = file_uses_sha256[file_ref.sha256]; + if (uses === undefined) { + uses = await idb_get(transaction, "file_uses", file_ref.sha256); + if (uses) + [uses.new, uses.initial] = [false, uses.uses]; + else + uses = {sha256: file_ref.sha256, uses: 0, new: true, initial: 0}; + + file_uses_sha256[file_ref.sha256] = uses; + } + + return uses; +} + +/* + * Save given definition of a resource/mapping to IndexedDB. If the definition + * (passed as `item`) references files that are not already present in + * IndexedDB, those files should be present as values of the `files_sha256` + * object with keys being their sha256 sums. + */ +async function save_item(item, files_sha256, db) +{ + const store_name = {resource: "resources", mapping: "mappings"}[item.type]; + const transaction = + db.transaction([store_name, "files", "file_uses"], "readwrite"); + + let resolve, reject; + const result = new Promise((...cbs) => [resolve, reject] = cbs); + transaction.oncomplete = resolve; + transaction.onerror = reject; + + const uses_sha256 = {}; + for (const file_ref of entities.get_files(item)) + (await get_file_uses(transaction, uses_sha256, file_ref)).uses++; + + const old_item = await idb_get(transaction, store_name, item.identifier); + if (old_item !== undefined) { + for (const file_ref of entities.get_files(old_item)) + (await get_file_uses(transaction, uses_sha256, file_ref)).uses--; + } + + for (const uses of Object.values(uses_sha256)) { + if (uses.uses < 0) + console.error("internal error: uses < 0 for file " + uses.sha256); + + const [is_new, initial_uses] = [uses.new, uses.initial]; + delete uses.new; + delete uses.initial; + + if (uses.uses < 1) { + if (!is_new) { + idb_del(transaction, "file_uses", uses.sha256); + idb_del(transaction, "files", uses.sha256); + } + + continue; + } + + if (uses.uses === initial_uses) + continue; + + const file = files_sha256[uses.sha256]; + if (file === undefined) + throw "file not present: " + uses.sha256; + + idb_put(transaction, "files", {sha256: uses.sha256, contents: file}); + idb_put(transaction, "file_uses", uses); + } + + idb_put(transaction, store_name, item); + + return result; +} + +const haketilodb = { + get: get_db, + save_item: save_item +}; + +/* + * EXPORTS_START + * EXPORT haketilodb + * EXPORT idb_get + * EXPORT idb_put + * EXPORT idb_del + * EXPORTS_END + */ diff --git a/compute_scripts.awk b/compute_scripts.awk index 1f3b11e..3f237e1 100644 --- a/compute_scripts.awk +++ b/compute_scripts.awk @@ -163,9 +163,10 @@ function print_usage() { } function mock_exports_init() { - provides["browser"] = "exports_init.js" - provides["is_chrome"] = "exports_init.js" - provides["is_mozilla"] = "exports_init.js" + provides["browser"] = "exports_init.js" + provides["is_chrome"] = "exports_init.js" + provides["is_mozilla"] = "exports_init.js" + provides["initial_data"] = "exports_init.js" processed["exports_init.js"] = "used" } diff --git a/test/script_loader.py b/test/script_loader.py index 15269c7..8f30944 100644 --- a/test/script_loader.py +++ b/test/script_loader.py @@ -51,7 +51,12 @@ def available_scripts(directory): def wrapped_script(script_path, wrap_partially=True): if script_path == 'exports_init.js': - with open(script_root / 'MOZILLA_exports_init.js') as script: + if not (script_root / 'exports_init.js').exists(): + subprocess.run([str(script_root / 'write_exports_init.sh'), + 'mozilla', '.', 'default_settings.json'], + cwd=script_root, check=True) + + with open(script_root / 'exports_init.js') as script: return script.read() command = 'partially_wrapped_code' if wrap_partially else 'wrapped_code' diff --git a/test/unit/test_indexeddb.py b/test/unit/test_indexeddb.py new file mode 100644 index 0000000..e5e1626 --- /dev/null +++ b/test/unit/test_indexeddb.py @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - IndexedDB access +""" + +# This file is part of Haketilo +# +# Copyright (C) 2021, Wojtek Kosior <koszko@koszko.org> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +from hashlib import sha256 + +from ..script_loader import load_script + +@pytest.fixture(scope="session") +def indexeddb_code(): + yield load_script('common/indexeddb.js', ['common']) + +def sample_file(contents): + return { + 'sha256': sha256(contents.encode()).digest().hex(), + contents: contents + } + +sample_files = { + 'report.spdx': sample_file('<!-- dummy report -->'), + 'LICENSES/somelicense.txt': sample_file('Permission is granted...'), + 'hello.js': sample_file('console.log("hello!");\n'), + 'bye.js': sample_file('console.log("bye!");\n'), + 'README.md': sample_file('# Python Frobnicator\n...') +} + +sample_files_sha256 = \ + dict([[file['sha256'], file] for file in sample_files.values()]) + +def file_ref(file_name): + return {'file': file_name, 'sha256': sample_files[file_name]['sha256']} + +def test_save_item(execute_in_page, indexeddb_code): + """ + indexeddb.js facilitates operating on Haketilo's internal database. + Verify database operations work properly. + """ + execute_in_page(indexeddb_code, page='https://gotmyowndoma.in') + # Don't use Haketilo's default initial data. + execute_in_page( + '''{ + const _get_db = haketilodb.get; + get_db = () => _get_db({}); + haketilodb.get = get_db; + }''' + ) + + # Start with no database. + execute_in_page( + '''{ + async function delete_db() { + let resolve; + const result = new Promise(_resolve => resolve = _resolve); + const request = indexedDB.deleteDatabase("haketilo"); + [request.onsuccess, request.onerror] = [resolve, resolve]; + await result; + } + + returnval(delete_db()); + }''' + ) + + # Facilitate retrieving all IndexedDB contents. + execute_in_page( + ''' + async function get_database_contents(promise=Promise.resolve()) + { + if (promise) + await promise; + + const db = await haketilodb.get(); + + const transaction = db.transaction(db.objectStoreNames); + const store_names_reqs = [...db.objectStoreNames] + .map(sn => [sn, transaction.objectStore(sn).getAll()]) + + const promises = store_names_reqs + .map(([_, req]) => wait_request(req)); + await Promise.all(promises); + + const result = {}; + store_names_reqs.forEach(([sn, req]) => result[sn] = req.result); + return result; + } + ''') + + # Sample resource definition. It'd normally contain more fields but here + # we use a simplified version. + sample_item = { + 'source_copyright': [ + file_ref('report.spdx'), + file_ref('LICENSES/somelicense.txt') + ], + 'type': 'resource', + 'identifier': 'helloapple', + 'scripts': [file_ref('hello.js'), file_ref('bye.js')], + 'type': 'resource' + } + next(iter(sample_item['source_copyright']))['ugly_extra_property'] = True + + database_contents = execute_in_page( + '''{ + const prom = haketilodb.get().then(db => save_item(...arguments, db)); + returnval(get_database_contents(prom)); + }''', + sample_item, sample_files_sha256) + assert len(database_contents['files']) == 4 + assert all([sample_files_sha256[file['sha256']] == file['contents'] + for file in database_contents['files']]) + assert all([len(file) == 2 for file in database_contents['files']]) + + assert len(database_contents['file_uses']) == 4 + assert all([uses['uses'] == 1 for uses in database_contents['file_uses']]) + assert set([uses['sha256'] for uses in database_contents['file_uses']]) \ + == set([file['sha256'] for file in database_contents['files']]) + + assert database_contents['mappings'] == [] + assert database_contents['resources'] == [sample_item] diff --git a/write_exports_init.sh b/write_exports_init.sh new file mode 100755 index 0000000..d434e94 --- /dev/null +++ b/write_exports_init.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# This file is part of Haketilo +# +# Copyright (C) 2021, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +set -e + +BROWSER="$1" +BUILDDIR="$2" +SETTINGS="$3" + +if [ "chromium" = "$BROWSER" ]; then + cp CHROMIUM_exports_init.js "$BUILDDIR"/exports_init.js +else + cp MOZILLA_exports_init.js "$BUILDDIR"/exports_init.js +fi + +printf 'window.haketilo_exports.initial_data = %s;\n' "$(cat "$SETTINGS")" \ + >> "$BUILDDIR"/exports_init.js |