aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Makefile.in2
-rw-r--r--background/main.js13
-rwxr-xr-xbuild.sh11
-rw-r--r--common/entities.js148
-rw-r--r--common/indexeddb.js270
-rw-r--r--compute_scripts.awk7
-rw-r--r--test/script_loader.py7
-rw-r--r--test/unit/test_indexeddb.py134
-rwxr-xr-xwrite_exports_init.sh29
9 files changed, 598 insertions, 23 deletions
diff --git a/Makefile.in b/Makefile.in
index 5a376e1..5291299 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -79,7 +79,7 @@ test-environment: test/certs/rootCA.pem test/certs/site.key
clean mostlyclean:
rm -rf mozilla-unpacked chromium-unpacked haketilo-$(version)
rm -f mozilla-build.zip chromium-build.zip haketilo-$(version).tar.gz \
- haketilo-$(version).tar
+ haketilo-$(version).tar exports_init.js
rm -rf test/certs
rm -rf $$(find . -name geckodriver.log)
rm -rf $$(find . -type d -name __pycache__)
diff --git a/background/main.js b/background/main.js
index a4d3f0e..2809334 100644
--- a/background/main.js
+++ b/background/main.js
@@ -3,7 +3,8 @@
*
* Function: Main background script.
*
- * Copyright (C) 2021 Wojtek Kosior
+ * Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org>
+ * Copyright (C) 2021 Jahoti <jahoti@envs.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -43,6 +44,7 @@
/*
* IMPORTS_START
+ * IMPORT initial_data
* IMPORT TYPE_PREFIX
* IMPORT get_storage
* IMPORT light_storage
@@ -70,13 +72,8 @@ async function init_ext(install_details)
await storage.clear();
- /*
- * Below we add sample settings to the extension.
- */
-
- for (let setting of // The next line is replaced with the contents of /default_settings.json by the build script
- `DEFAULT SETTINGS`
- ) {
+ /* Below we add sample settings to the extension. */
+ for (let setting of initial_data) {
let [key, value] = Object.entries(setting)[0];
storage.set(key[0], key.substring(1), value);
}
diff --git a/build.sh b/build.sh
index abaed47..d0958a1 100755
--- a/build.sh
+++ b/build.sh
@@ -123,16 +123,7 @@ s^_CONTENTSCRIPTS_^$CONTENTSCRIPTS^" \
fi
done
- # A hack to insert the contents of default_settings.json at the appropriate
- # location in background/main.js. Uses an internal sed expression to escape
- # and indent the JSON file for use in the external sed expression.
- sed -i 's/^ `DEFAULT SETTINGS`$/'"$(sed -E 's/([\\\&\/])/\\\1/g; s/^/ /; s/$/\\/' < default_settings.json) "/g "$BUILDDIR"/background/main.js
-
- if [ "$BROWSER" = "chromium" ]; then
- cp CHROMIUM_exports_init.js "$BUILDDIR"/exports_init.js
- else
- cp MOZILLA_exports_init.js "$BUILDDIR"/exports_init.js
- fi
+ ./write_exports_init.sh "$BROWSER" "$BUILDDIR" default_settings.json
cp -r copyright licenses/ "$BUILDDIR"
cp dummy "$BUILDDIR"
diff --git a/common/entities.js b/common/entities.js
new file mode 100644
index 0000000..46836b5
--- /dev/null
+++ b/common/entities.js
@@ -0,0 +1,148 @@
+/**
+ * This file is part of Haketilo.
+ *
+ * Function: Operations on resources and mappings.
+ *
+ * Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * As additional permission under GNU GPL version 3 section 7, you
+ * may distribute forms of that code without the copy of the GNU
+ * GPL normally required by section 4, provided you include this
+ * license notice and, in case of non-source distribution, a URL
+ * through which recipients can access the Corresponding Source.
+ * If you modify file(s) with this exception, you may extend this
+ * exception to your version of the file(s), but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * As a special exception to the GPL, any HTML file which merely
+ * makes function calls to this code, and for that purpose
+ * includes it by reference shall be deemed a separate work for
+ * copyright law purposes. If you modify this code, you may extend
+ * this exception to your version of the code, but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * I, Wojtek Kosior, thereby promise not to sue for violation of this file's
+ * license. Although I request that you do not make use this code in a
+ * proprietary program, I am not going to enforce this in court.
+ */
+
+/*
+ * Convert ver_str into an array representation, e.g. for ver_str="4.6.13.0"
+ * return [4, 6, 13, 0].
+ */
+const parse_version = ver_str => ver_str.split(".").map(parseInt);
+
+/*
+ * ver is an array of integers. rev is an optional integer. Produce string
+ * representation of version (optionally with revision number), like:
+ * 1.2.3-5
+ * No version normalization is performed.
+ */
+const version_string = (ver, rev=0) => ver.join(".") + (rev ? `-${rev}` : "");
+
+/* vers should be an array of comparable values. Return the greatest one. */
+const max = vals => Array.reduce(vals, (v1, v2) => v1 > v2 ? v1 : v2);
+
+/*
+ * versioned_item should be a dict with keys being version strings and values
+ * being definitions of the respective versions of a single resource/mapping.
+ * Example:
+ * {
+ * "1": {
+ * version: [1]//,
+ * // more stuff
+ * },
+ * "1.1": {
+ * version: [1, 1]//,
+ * // more stuff
+ * }
+ * }
+ *
+ * Returns the definition with the highest version.
+ */
+function get_newest_version(versioned_item)
+{
+ const best_ver = max(Object.keys(versioned_item).map(parse_version));
+ return versioned_item[version_string(best_ver)];
+}
+
+/*
+ * item is a definition of a resource or mapping. Yield all file references
+ * (objects with `file` and `sha256` properties) this definition has.
+ */
+function* get_used_files(item)
+{
+ for (const file of item.source_copyright)
+ yield file;
+
+ if (item.type === "resource") {
+ for (const file of item.scripts || [])
+ yield file;
+ }
+}
+
+const entities = {
+ get_newest: get_newest_version,
+ get_files: get_used_files
+};
+
+/*
+ * EXPORTS_START
+ * EXPORT entities
+ * EXPORTS_END
+ */
+
+/*
+ * Note: the functions below were overeagerly written and are not used now but
+ * might prove useful to once we add more functionalities and are hence kept...
+ */
+
+/*
+ * Clone recursively all objects. Leave other items (arrays, strings) untouched.
+ */
+function deep_object_copy(object)
+{
+ const orig = {object};
+ const result = {};
+ const to_copy = [[orig, {}]];
+
+ while (to_copy.length > 0) {
+ const [object, copy] = to_copy.pop();
+
+ for (const [key, value] of Object.entries(object)) {
+ copy[key] = value;
+
+ if (typeof value === "object" && !Array.isArray(value)) {
+ const value_copy = {};
+ to_copy.push([value, value_copy]);
+ copy[key] = value_copy;
+ }
+ }
+ }
+
+ return result.orig;
+}
+
+/* helper function for normalize_version() */
+const version_reductor = (acc, n) => [...(n || acc.length ? [n] : []), ...acc];
+/*
+ * ver is an array of integers. Strip right-most zeroes from ver.
+ *
+ * Returns a *new* array. Doesn't modify its argument.
+ */
+const normalize_version = ver => Array.reduceRight(ver, version_reductor, []);
diff --git a/common/indexeddb.js b/common/indexeddb.js
new file mode 100644
index 0000000..d0bacc4
--- /dev/null
+++ b/common/indexeddb.js
@@ -0,0 +1,270 @@
+/**
+ * This file is part of Haketilo.
+ *
+ * Function: Facilitate use of IndexedDB within Haketilo.
+ *
+ * Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * As additional permission under GNU GPL version 3 section 7, you
+ * may distribute forms of that code without the copy of the GNU
+ * GPL normally required by section 4, provided you include this
+ * license notice and, in case of non-source distribution, a URL
+ * through which recipients can access the Corresponding Source.
+ * If you modify file(s) with this exception, you may extend this
+ * exception to your version of the file(s), but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * As a special exception to the GPL, any HTML file which merely
+ * makes function calls to this code, and for that purpose
+ * includes it by reference shall be deemed a separate work for
+ * copyright law purposes. If you modify this code, you may extend
+ * this exception to your version of the code, but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * I, Wojtek Kosior, thereby promise not to sue for violation of this file's
+ * license. Although I request that you do not make use this code in a
+ * proprietary program, I am not going to enforce this in court.
+ */
+
+/*
+ * IMPORTS_START
+ * IMPORT initial_data
+ * IMPORT entities
+ * IMPORTS_END
+ */
+
+/* Update when changes are made to database schema. Must have 3 elements */
+const db_version = [1, 0, 0];
+
+const nr_reductor = ([i, s], num) => [i - 1, s + num * 1024 ** i];
+const version_nr = ver => Array.reduce(ver.slice(0, 3), nr_reductor, [2, 0])[1];
+
+const stores = [
+ ["files", {keyPath: "sha256"}],
+ ["file_uses", {keyPath: "sha256"}],
+ ["resources", {keyPath: "identifier"}],
+ ["mappings", {keyPath: "identifier"}]
+];
+
+let db = null;
+
+/* Generate a Promise that resolves when an IndexedDB request succeeds. */
+async function wait_request(idb_request)
+{
+ let resolve, reject;
+ const waiter = new Promise((...cbs) => [resolve, reject] = cbs);
+ [idb_request.onsuccess, idb_request.onerror] = [resolve, reject];
+ return waiter;
+}
+
+/* asynchronous wrapper for IDBObjectStore's get() method. */
+async function idb_get(transaction, store_name, key)
+{
+ const req = transaction.objectStore(store_name).get(key);
+ return (await wait_request(req)).target.result;
+}
+
+/* asynchronous wrapper for IDBObjectStore's put() method. */
+async function idb_put(transaction, store_name, object)
+{
+ return wait_request(transaction.objectStore(store_name).put(object));
+}
+
+/* asynchronous wrapper for IDBObjectStore's delete() method. */
+async function idb_del(transaction, store_name, key)
+{
+ return wait_request(transaction.objectStore(store_name).delete(key));
+}
+
+/* Open haketilo database, asynchronously return an IDBDatabase object. */
+async function get_db(initialization_data=initial_data)
+{
+ if (db)
+ return db;
+
+ let resolve, reject;
+ const waiter = new Promise((...cbs) => [resolve, reject] = cbs);
+
+ const request = indexedDB.open("haketilo", version_nr(db_version));
+ request.onsuccess = resolve;
+ request.onerror = ev => reject("db error: " + ev.target.errorCode);
+ request.onupgradeneeded = resolve;
+
+ const event = await waiter;
+ const opened_db = event.target.result;
+
+ if (event instanceof IDBVersionChangeEvent) {
+ /*
+ * When we move to a new database schema, we will add upgrade logic
+ * here.
+ */
+ if (event.oldVersion > 0)
+ throw "bad db version: " + event.oldVersion;
+
+ let store;
+ for (const [store_name, key_mode] of stores)
+ store = opened_db.createObjectStore(store_name, key_mode);
+
+ await new Promise(resolve => store.transaction.oncomplete = resolve);
+
+ save_items(db, initialization_data);
+ }
+
+ db = opened_db;
+
+ return db;
+}
+
+/*
+ * How a sample data argument to the function below might look like:
+ *
+ * data = {
+ * resources: {
+ * "resource1": {
+ * "1": {
+ * // some stuff
+ * },
+ * "1.1": {
+ * // some stuff
+ * }
+ * },
+ * "resource2": {
+ * "0.4.3": {
+ * // some stuff
+ * }
+ * },
+ * },
+ * mappings: {
+ * "mapping1": {
+ * "2": {
+ * // some stuff
+ * }
+ * },
+ * "mapping2": {
+ * "0.1": {
+ * // some stuff
+ * }
+ * },
+ * },
+ * files: {
+ * "sha256-f9444510dc7403e41049deb133f6892aa6a63c05591b2b59e4ee5b234d7bbd99": "console.log(\"hello\");\n",
+ * "sha256-b857cd521cc82fff30f0d316deba38b980d66db29a5388eb6004579cf743c6fd": "console.log(\"bye\");"
+ * }
+ * }
+ */
+async function save_items(db, data)
+{
+ const files = data.files;
+ const resources =
+ Object.values(data.resources || []).map(entities.get_newest);
+ const mappings =
+ Object.values(data.mappings || []).map(entities.get_newest);
+
+ resources.concat(mappings).forEach(i => save_item(i, data.files, db));
+}
+
+/* helper function of save_item() */
+async function get_file_uses(transaction, file_uses_sha256, file_ref)
+{
+ let uses = file_uses_sha256[file_ref.sha256];
+ if (uses === undefined) {
+ uses = await idb_get(transaction, "file_uses", file_ref.sha256);
+ if (uses)
+ [uses.new, uses.initial] = [false, uses.uses];
+ else
+ uses = {sha256: file_ref.sha256, uses: 0, new: true, initial: 0};
+
+ file_uses_sha256[file_ref.sha256] = uses;
+ }
+
+ return uses;
+}
+
+/*
+ * Save given definition of a resource/mapping to IndexedDB. If the definition
+ * (passed as `item`) references files that are not already present in
+ * IndexedDB, those files should be present as values of the `files_sha256`
+ * object with keys being their sha256 sums.
+ */
+async function save_item(item, files_sha256, db)
+{
+ const store_name = {resource: "resources", mapping: "mappings"}[item.type];
+ const transaction =
+ db.transaction([store_name, "files", "file_uses"], "readwrite");
+
+ let resolve, reject;
+ const result = new Promise((...cbs) => [resolve, reject] = cbs);
+ transaction.oncomplete = resolve;
+ transaction.onerror = reject;
+
+ const uses_sha256 = {};
+ for (const file_ref of entities.get_files(item))
+ (await get_file_uses(transaction, uses_sha256, file_ref)).uses++;
+
+ const old_item = await idb_get(transaction, store_name, item.identifier);
+ if (old_item !== undefined) {
+ for (const file_ref of entities.get_files(old_item))
+ (await get_file_uses(transaction, uses_sha256, file_ref)).uses--;
+ }
+
+ for (const uses of Object.values(uses_sha256)) {
+ if (uses.uses < 0)
+ console.error("internal error: uses < 0 for file " + uses.sha256);
+
+ const [is_new, initial_uses] = [uses.new, uses.initial];
+ delete uses.new;
+ delete uses.initial;
+
+ if (uses.uses < 1) {
+ if (!is_new) {
+ idb_del(transaction, "file_uses", uses.sha256);
+ idb_del(transaction, "files", uses.sha256);
+ }
+
+ continue;
+ }
+
+ if (uses.uses === initial_uses)
+ continue;
+
+ const file = files_sha256[uses.sha256];
+ if (file === undefined)
+ throw "file not present: " + uses.sha256;
+
+ idb_put(transaction, "files", {sha256: uses.sha256, contents: file});
+ idb_put(transaction, "file_uses", uses);
+ }
+
+ idb_put(transaction, store_name, item);
+
+ return result;
+}
+
+const haketilodb = {
+ get: get_db,
+ save_item: save_item
+};
+
+/*
+ * EXPORTS_START
+ * EXPORT haketilodb
+ * EXPORT idb_get
+ * EXPORT idb_put
+ * EXPORT idb_del
+ * EXPORTS_END
+ */
diff --git a/compute_scripts.awk b/compute_scripts.awk
index 1f3b11e..3f237e1 100644
--- a/compute_scripts.awk
+++ b/compute_scripts.awk
@@ -163,9 +163,10 @@ function print_usage() {
}
function mock_exports_init() {
- provides["browser"] = "exports_init.js"
- provides["is_chrome"] = "exports_init.js"
- provides["is_mozilla"] = "exports_init.js"
+ provides["browser"] = "exports_init.js"
+ provides["is_chrome"] = "exports_init.js"
+ provides["is_mozilla"] = "exports_init.js"
+ provides["initial_data"] = "exports_init.js"
processed["exports_init.js"] = "used"
}
diff --git a/test/script_loader.py b/test/script_loader.py
index 15269c7..8f30944 100644
--- a/test/script_loader.py
+++ b/test/script_loader.py
@@ -51,7 +51,12 @@ def available_scripts(directory):
def wrapped_script(script_path, wrap_partially=True):
if script_path == 'exports_init.js':
- with open(script_root / 'MOZILLA_exports_init.js') as script:
+ if not (script_root / 'exports_init.js').exists():
+ subprocess.run([str(script_root / 'write_exports_init.sh'),
+ 'mozilla', '.', 'default_settings.json'],
+ cwd=script_root, check=True)
+
+ with open(script_root / 'exports_init.js') as script:
return script.read()
command = 'partially_wrapped_code' if wrap_partially else 'wrapped_code'
diff --git a/test/unit/test_indexeddb.py b/test/unit/test_indexeddb.py
new file mode 100644
index 0000000..e5e1626
--- /dev/null
+++ b/test/unit/test_indexeddb.py
@@ -0,0 +1,134 @@
+# SPDX-License-Identifier: CC0-1.0
+
+"""
+Haketilo unit tests - IndexedDB access
+"""
+
+# This file is part of Haketilo
+#
+# Copyright (C) 2021, Wojtek Kosior <koszko@koszko.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the CC0 1.0 Universal License as published by
+# the Creative Commons Corporation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# CC0 1.0 Universal License for more details.
+
+import pytest
+from hashlib import sha256
+
+from ..script_loader import load_script
+
+@pytest.fixture(scope="session")
+def indexeddb_code():
+ yield load_script('common/indexeddb.js', ['common'])
+
+def sample_file(contents):
+ return {
+ 'sha256': sha256(contents.encode()).digest().hex(),
+ contents: contents
+ }
+
+sample_files = {
+ 'report.spdx': sample_file('<!-- dummy report -->'),
+ 'LICENSES/somelicense.txt': sample_file('Permission is granted...'),
+ 'hello.js': sample_file('console.log("hello!");\n'),
+ 'bye.js': sample_file('console.log("bye!");\n'),
+ 'README.md': sample_file('# Python Frobnicator\n...')
+}
+
+sample_files_sha256 = \
+ dict([[file['sha256'], file] for file in sample_files.values()])
+
+def file_ref(file_name):
+ return {'file': file_name, 'sha256': sample_files[file_name]['sha256']}
+
+def test_save_item(execute_in_page, indexeddb_code):
+ """
+ indexeddb.js facilitates operating on Haketilo's internal database.
+ Verify database operations work properly.
+ """
+ execute_in_page(indexeddb_code, page='https://gotmyowndoma.in')
+ # Don't use Haketilo's default initial data.
+ execute_in_page(
+ '''{
+ const _get_db = haketilodb.get;
+ get_db = () => _get_db({});
+ haketilodb.get = get_db;
+ }'''
+ )
+
+ # Start with no database.
+ execute_in_page(
+ '''{
+ async function delete_db() {
+ let resolve;
+ const result = new Promise(_resolve => resolve = _resolve);
+ const request = indexedDB.deleteDatabase("haketilo");
+ [request.onsuccess, request.onerror] = [resolve, resolve];
+ await result;
+ }
+
+ returnval(delete_db());
+ }'''
+ )
+
+ # Facilitate retrieving all IndexedDB contents.
+ execute_in_page(
+ '''
+ async function get_database_contents(promise=Promise.resolve())
+ {
+ if (promise)
+ await promise;
+
+ const db = await haketilodb.get();
+
+ const transaction = db.transaction(db.objectStoreNames);
+ const store_names_reqs = [...db.objectStoreNames]
+ .map(sn => [sn, transaction.objectStore(sn).getAll()])
+
+ const promises = store_names_reqs
+ .map(([_, req]) => wait_request(req));
+ await Promise.all(promises);
+
+ const result = {};
+ store_names_reqs.forEach(([sn, req]) => result[sn] = req.result);
+ return result;
+ }
+ ''')
+
+ # Sample resource definition. It'd normally contain more fields but here
+ # we use a simplified version.
+ sample_item = {
+ 'source_copyright': [
+ file_ref('report.spdx'),
+ file_ref('LICENSES/somelicense.txt')
+ ],
+ 'type': 'resource',
+ 'identifier': 'helloapple',
+ 'scripts': [file_ref('hello.js'), file_ref('bye.js')],
+ 'type': 'resource'
+ }
+ next(iter(sample_item['source_copyright']))['ugly_extra_property'] = True
+
+ database_contents = execute_in_page(
+ '''{
+ const prom = haketilodb.get().then(db => save_item(...arguments, db));
+ returnval(get_database_contents(prom));
+ }''',
+ sample_item, sample_files_sha256)
+ assert len(database_contents['files']) == 4
+ assert all([sample_files_sha256[file['sha256']] == file['contents']
+ for file in database_contents['files']])
+ assert all([len(file) == 2 for file in database_contents['files']])
+
+ assert len(database_contents['file_uses']) == 4
+ assert all([uses['uses'] == 1 for uses in database_contents['file_uses']])
+ assert set([uses['sha256'] for uses in database_contents['file_uses']]) \
+ == set([file['sha256'] for file in database_contents['files']])
+
+ assert database_contents['mappings'] == []
+ assert database_contents['resources'] == [sample_item]
diff --git a/write_exports_init.sh b/write_exports_init.sh
new file mode 100755
index 0000000..d434e94
--- /dev/null
+++ b/write_exports_init.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# This file is part of Haketilo
+#
+# Copyright (C) 2021, Wojtek Kosior
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the CC0 1.0 Universal License as published by
+# the Creative Commons Corporation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# CC0 1.0 Universal License for more details.
+
+set -e
+
+BROWSER="$1"
+BUILDDIR="$2"
+SETTINGS="$3"
+
+if [ "chromium" = "$BROWSER" ]; then
+ cp CHROMIUM_exports_init.js "$BUILDDIR"/exports_init.js
+else
+ cp MOZILLA_exports_init.js "$BUILDDIR"/exports_init.js
+fi
+
+printf 'window.haketilo_exports.initial_data = %s;\n' "$(cat "$SETTINGS")" \
+ >> "$BUILDDIR"/exports_init.js