facilitate initialization of IndexedDB for use by Haketilo

9 files changed, 598 insertions, 23 deletions

diff --git a/Makefile.in b/Makefile.in
index 5a376e1..5291299 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -79,7 +79,7 @@ test-environment: test/certs/rootCA.pem test/certs/site.key
 clean mostlyclean:
 	rm -rf mozilla-unpacked chromium-unpacked haketilo-$(version)
 	rm -f mozilla-build.zip chromium-build.zip haketilo-$(version).tar.gz \
-	        haketilo-$(version).tar
+	        haketilo-$(version).tar exports_init.js
 	rm -rf test/certs
 	rm -rf $$(find . -name geckodriver.log)
 	rm -rf $$(find . -type d -name __pycache__)
diff --git a/background/main.js b/background/main.js
index a4d3f0e..2809334 100644
--- a/background/main.js
+++ b/background/main.js
@@ -3,7 +3,8 @@
  *
  * Function: Main background script.
  *
- * Copyright (C) 2021 Wojtek Kosior
+ * Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org>
+ * Copyright (C) 2021 Jahoti <jahoti@envs.net>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -43,6 +44,7 @@
 
 /*
  * IMPORTS_START
+ * IMPORT initial_data
  * IMPORT TYPE_PREFIX
  * IMPORT get_storage
  * IMPORT light_storage
@@ -70,13 +72,8 @@ async function init_ext(install_details)
 
     await storage.clear();
 
-    /*
-     * Below we add sample settings to the extension.
-     */
-
-    for (let setting of // The next line is replaced with the contents of /default_settings.json by the build script
-        `DEFAULT SETTINGS`
-    ) {
+    /* Below we add sample settings to the extension. */
+    for (let setting of initial_data) {
 	let [key, value] = Object.entries(setting)[0];
 	storage.set(key[0], key.substring(1), value);
     }
diff --git a/build.sh b/build.sh
index abaed47..d0958a1 100755
--- a/build.sh
+++ b/build.sh
@@ -123,16 +123,7 @@ s^_CONTENTSCRIPTS_^$CONTENTSCRIPTS^" \
 	fi
     done
 
-    # A hack to insert the contents of default_settings.json at the appropriate
-    # location in background/main.js. Uses an internal sed expression to escape
-    # and indent the JSON file for use in the external sed expression.
-    sed -i 's/^        `DEFAULT SETTINGS`$/'"$(sed -E 's/([\\\&\/])/\\\1/g; s/^/        /; s/$/\\/' < default_settings.json) "/g "$BUILDDIR"/background/main.js
-
-    if [ "$BROWSER" = "chromium" ]; then
-	cp CHROMIUM_exports_init.js "$BUILDDIR"/exports_init.js
-    else
-	cp MOZILLA_exports_init.js "$BUILDDIR"/exports_init.js
-    fi
+    ./write_exports_init.sh "$BROWSER" "$BUILDDIR" default_settings.json
 
     cp -r copyright licenses/ "$BUILDDIR"
     cp dummy "$BUILDDIR"
diff --git a/common/entities.js b/common/entities.js
new file mode 100644
index 0000000..46836b5
--- /dev/null
+++ b/common/entities.js
@@ -0,0 +1,148 @@
+/**
+ * This file is part of Haketilo.
+ *
+ * Function: Operations on resources and mappings.
+ *
+ * Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * As additional permission under GNU GPL version 3 section 7, you
+ * may distribute forms of that code without the copy of the GNU
+ * GPL normally required by section 4, provided you include this
+ * license notice and, in case of non-source distribution, a URL
+ * through which recipients can access the Corresponding Source.
+ * If you modify file(s) with this exception, you may extend this
+ * exception to your version of the file(s), but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * As a special exception to the GPL, any HTML file which merely
+ * makes function calls to this code, and for that purpose
+ * includes it by reference shall be deemed a separate work for
+ * copyright law purposes. If you modify this code, you may extend
+ * this exception to your version of the code, but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * I, Wojtek Kosior, thereby promise not to sue for violation of this file's
+ * license. Although I request that you do not make use this code in a
+ * proprietary program, I am not going to enforce this in court.
+ */
+
+/*
+ * Convert ver_str into an array representation, e.g. for ver_str="4.6.13.0"
+ * return [4, 6, 13, 0].
+ */
+const parse_version = ver_str => ver_str.split(".").map(parseInt);
+
+/*
+ * ver is an array of integers. rev is an optional integer. Produce string
+ * representation of version (optionally with revision number), like:
+ *     1.2.3-5
+ * No version normalization is performed.
+ */
+const version_string = (ver, rev=0) => ver.join(".") + (rev ? `-${rev}` : "");
+
+/* vers should be an array of comparable values. Return the greatest one. */
+const max = vals => Array.reduce(vals, (v1, v2) => v1 > v2 ? v1 : v2);
+
+/*
+ * versioned_item should be a dict with keys being version strings and values
+ * being definitions of the respective versions of a single resource/mapping.
+ * Example:
+ *     {
+ *         "1": {
+ *             version: [1]//,
+ *             // more stuff
+ *         },
+ *         "1.1": {
+ *             version: [1, 1]//,
+ *             // more stuff
+ *         }
+ *      }
+ *
+ * Returns the definition with the highest version.
+ */
+function get_newest_version(versioned_item)
+{
+    const best_ver = max(Object.keys(versioned_item).map(parse_version));
+    return versioned_item[version_string(best_ver)];
+}
+
+/*
+ * item is a definition of a resource or mapping. Yield all file references
+ * (objects with `file` and `sha256` properties) this definition has.
+ */
+function* get_used_files(item)
+{
+    for (const file of item.source_copyright)
+	yield file;
+
+    if (item.type === "resource") {
+	for (const file of item.scripts || [])
+	    yield file;
+    }
+}
+
+const entities = {
+    get_newest: get_newest_version,
+    get_files:  get_used_files
+};
+
+/*
+ * EXPORTS_START
+ * EXPORT entities
+ * EXPORTS_END
+ */
+
+/*
+ * Note: the functions below were overeagerly written and are not used now but
+ * might prove useful to once we add more functionalities and are hence kept...
+ */
+
+/*
+ * Clone recursively all objects. Leave other items (arrays, strings) untouched.
+ */
+function deep_object_copy(object)
+{
+    const orig = {object};
+    const result = {};
+    const to_copy = [[orig, {}]];
+
+    while (to_copy.length > 0) {
+	const [object, copy] = to_copy.pop();
+
+	for (const [key, value] of Object.entries(object)) {
+	    copy[key] = value;
+
+	    if (typeof value === "object" && !Array.isArray(value)) {
+		const value_copy = {};
+		to_copy.push([value, value_copy]);
+		copy[key] = value_copy;
+	    }
+	}
+    }
+
+    return result.orig;
+}
+
+/* helper function for normalize_version() */
+const version_reductor = (acc, n) => [...(n || acc.length ? [n] : []), ...acc];
+/*
+ * ver is an array of integers. Strip right-most zeroes from ver.
+ *
+ * Returns a *new* array. Doesn't modify its argument.
+ */
+const normalize_version = ver => Array.reduceRight(ver, version_reductor, []);
diff --git a/common/indexeddb.js b/common/indexeddb.js
new file mode 100644
index 0000000..d0bacc4
--- /dev/null
+++ b/common/indexeddb.js
@@ -0,0 +1,270 @@
+/**
+ * This file is part of Haketilo.
+ *
+ * Function: Facilitate use of IndexedDB within Haketilo.
+ *
+ * Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * As additional permission under GNU GPL version 3 section 7, you
+ * may distribute forms of that code without the copy of the GNU
+ * GPL normally required by section 4, provided you include this
+ * license notice and, in case of non-source distribution, a URL
+ * through which recipients can access the Corresponding Source.
+ * If you modify file(s) with this exception, you may extend this
+ * exception to your version of the file(s), but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * As a special exception to the GPL, any HTML file which merely
+ * makes function calls to this code, and for that purpose
+ * includes it by reference shall be deemed a separate work for
+ * copyright law purposes. If you modify this code, you may extend
+ * this exception to your version of the code, but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * I, Wojtek Kosior, thereby promise not to sue for violation of this file's
+ * license. Although I request that you do not make use this code in a
+ * proprietary program, I am not going to enforce this in court.
+ */
+
+/*
+ * IMPORTS_START
+ * IMPORT initial_data
+ * IMPORT entities
+ * IMPORTS_END
+ */
+
+/* Update when changes are made to database schema. Must have 3 elements */
+const db_version = [1, 0, 0];
+
+const nr_reductor = ([i, s], num) => [i - 1, s + num * 1024 ** i];
+const version_nr = ver => Array.reduce(ver.slice(0, 3), nr_reductor, [2, 0])[1];
+
+const stores = 	[
+    ["files",     {keyPath: "sha256"}],
+    ["file_uses", {keyPath: "sha256"}],
+    ["resources", {keyPath: "identifier"}],
+    ["mappings",  {keyPath: "identifier"}]
+];
+
+let db = null;
+
+/* Generate a Promise that resolves when an IndexedDB request succeeds. */
+async function wait_request(idb_request)
+{
+    let resolve, reject;
+    const waiter = new Promise((...cbs) => [resolve, reject] = cbs);
+    [idb_request.onsuccess, idb_request.onerror] = [resolve, reject];
+    return waiter;
+}
+
+/* asynchronous wrapper for IDBObjectStore's get() method. */
+async function idb_get(transaction, store_name, key)
+{
+    const req = transaction.objectStore(store_name).get(key);
+    return (await wait_request(req)).target.result;
+}
+
+/* asynchronous wrapper for IDBObjectStore's put() method. */
+async function idb_put(transaction, store_name, object)
+{
+    return wait_request(transaction.objectStore(store_name).put(object));
+}
+
+/* asynchronous wrapper for IDBObjectStore's delete() method. */
+async function idb_del(transaction, store_name, key)
+{
+    return wait_request(transaction.objectStore(store_name).delete(key));
+}
+
+/* Open haketilo database, asynchronously return an IDBDatabase object. */
+async function get_db(initialization_data=initial_data)
+{
+    if (db)
+	return db;
+
+    let resolve, reject;
+    const waiter = new Promise((...cbs) => [resolve, reject] = cbs);
+
+    const request = indexedDB.open("haketilo", version_nr(db_version));
+    request.onsuccess       = resolve;
+    request.onerror         = ev => reject("db error: " + ev.target.errorCode);
+    request.onupgradeneeded = resolve;
+
+    const event = await waiter;
+    const opened_db = event.target.result;
+
+    if (event instanceof IDBVersionChangeEvent) {
+	/*
+	 * When we move to a new database schema, we will add upgrade logic
+	 * here.
+	 */
+	if (event.oldVersion > 0)
+	    throw "bad db version: " + event.oldVersion;
+
+	let store;
+	for (const [store_name, key_mode] of stores)
+	    store = opened_db.createObjectStore(store_name, key_mode);
+
+	await new Promise(resolve => store.transaction.oncomplete = resolve);
+
+	save_items(db, initialization_data);
+    }
+
+    db = opened_db;
+
+    return db;
+}
+
+/*
+ * How a sample data argument to the function below might look like:
+ *
+ * data = {
+ *     resources: {
+ *         "resource1": {
+ *             "1": {
+ *                 // some stuff
+ *             },
+ *             "1.1": {
+ *                 // some stuff
+ *             }
+ *         },
+ *         "resource2": {
+ *             "0.4.3": {
+ *                 // some stuff
+ *             }
+ *         },
+ *     },
+ *     mappings: {
+ *         "mapping1": {
+ *             "2": {
+ *                 // some stuff
+ *             }
+ *         },
+ *         "mapping2": {
+ *             "0.1": {
+ *                 // some stuff
+ *             }
+ *         },
+ *     },
+ *     files: {
+ *         "sha256-f9444510dc7403e41049deb133f6892aa6a63c05591b2b59e4ee5b234d7bbd99": "console.log(\"hello\");\n",
+ *         "sha256-b857cd521cc82fff30f0d316deba38b980d66db29a5388eb6004579cf743c6fd": "console.log(\"bye\");"
+ *     }
+ * }
+ */
+async function save_items(db, data)
+{
+    const files = data.files;
+    const resources =
+	  Object.values(data.resources || []).map(entities.get_newest);
+    const mappings =
+	  Object.values(data.mappings || []).map(entities.get_newest);
+
+    resources.concat(mappings).forEach(i => save_item(i, data.files, db));
+}
+
+/* helper function of save_item() */
+async function get_file_uses(transaction, file_uses_sha256, file_ref)
+{
+    let uses = file_uses_sha256[file_ref.sha256];
+    if (uses === undefined) {
+	uses = await idb_get(transaction, "file_uses", file_ref.sha256);
+	if (uses)
+	    [uses.new, uses.initial] = [false, uses.uses];
+	else
+	    uses = {sha256: file_ref.sha256, uses: 0, new: true, initial: 0};
+
+	file_uses_sha256[file_ref.sha256] = uses;
+    }
+
+    return uses;
+}
+
+/*
+ * Save given definition of a resource/mapping to IndexedDB. If the definition
+ * (passed as `item`) references files that are not already present in
+ * IndexedDB, those files should be present as values of the `files_sha256`
+ * object with keys being their sha256 sums.
+ */
+async function save_item(item, files_sha256, db)
+{
+    const store_name = {resource: "resources", mapping: "mappings"}[item.type];
+    const transaction =
+	  db.transaction([store_name, "files", "file_uses"], "readwrite");
+
+    let resolve, reject;
+    const result = new Promise((...cbs) => [resolve, reject] = cbs);
+    transaction.oncomplete = resolve;
+    transaction.onerror = reject;
+
+    const uses_sha256 = {};
+    for (const file_ref of entities.get_files(item))
+	(await get_file_uses(transaction, uses_sha256, file_ref)).uses++;
+
+    const old_item = await idb_get(transaction, store_name, item.identifier);
+    if (old_item !== undefined) {
+	for (const file_ref of entities.get_files(old_item))
+	    (await get_file_uses(transaction, uses_sha256, file_ref)).uses--;
+    }
+
+    for (const uses of Object.values(uses_sha256)) {
+	if (uses.uses < 0)
+	    console.error("internal error: uses < 0 for file " + uses.sha256);
+
+	const [is_new, initial_uses] = [uses.new, uses.initial];
+	delete uses.new;
+	delete uses.initial;
+
+	if (uses.uses < 1) {
+	    if (!is_new) {
+		idb_del(transaction, "file_uses", uses.sha256);
+		idb_del(transaction, "files",     uses.sha256);
+	    }
+
+	    continue;
+	}
+
+	if (uses.uses === initial_uses)
+	    continue;
+
+	const file = files_sha256[uses.sha256];
+	if (file === undefined)
+	    throw "file not present: " + uses.sha256;
+
+	idb_put(transaction, "files", {sha256: uses.sha256, contents: file});
+	idb_put(transaction, "file_uses", uses);
+    }
+
+    idb_put(transaction, store_name, item);
+
+    return result;
+}
+
+const haketilodb = {
+    get:       get_db,
+    save_item: save_item
+};
+
+/*
+ * EXPORTS_START
+ * EXPORT haketilodb
+ * EXPORT idb_get
+ * EXPORT idb_put
+ * EXPORT idb_del
+ * EXPORTS_END
+ */
diff --git a/compute_scripts.awk b/compute_scripts.awk
index 1f3b11e..3f237e1 100644
--- a/compute_scripts.awk
+++ b/compute_scripts.awk
@@ -163,9 +163,10 @@ function print_usage() {
 }
 
 function mock_exports_init() {
-    provides["browser"]    = "exports_init.js"
-    provides["is_chrome"]  = "exports_init.js"
-    provides["is_mozilla"] = "exports_init.js"
+    provides["browser"]      = "exports_init.js"
+    provides["is_chrome"]    = "exports_init.js"
+    provides["is_mozilla"]   = "exports_init.js"
+    provides["initial_data"] = "exports_init.js"
 
     processed["exports_init.js"] = "used"
 }
diff --git a/test/script_loader.py b/test/script_loader.py
index 15269c7..8f30944 100644
--- a/test/script_loader.py
+++ b/test/script_loader.py
@@ -51,7 +51,12 @@ def available_scripts(directory):
 
 def wrapped_script(script_path, wrap_partially=True):
     if script_path == 'exports_init.js':
-        with open(script_root / 'MOZILLA_exports_init.js') as script:
+        if not (script_root / 'exports_init.js').exists():
+            subprocess.run([str(script_root / 'write_exports_init.sh'),
+                            'mozilla', '.', 'default_settings.json'],
+                           cwd=script_root, check=True)
+
+        with open(script_root / 'exports_init.js') as script:
             return script.read()
 
     command = 'partially_wrapped_code' if wrap_partially else 'wrapped_code'
diff --git a/test/unit/test_indexeddb.py b/test/unit/test_indexeddb.py
new file mode 100644
index 0000000..e5e1626
--- /dev/null
+++ b/test/unit/test_indexeddb.py
@@ -0,0 +1,134 @@
+# SPDX-License-Identifier: CC0-1.0
+
+"""
+Haketilo unit tests - IndexedDB access
+"""
+
+# This file is part of Haketilo
+#
+# Copyright (C) 2021, Wojtek Kosior <koszko@koszko.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the CC0 1.0 Universal License as published by
+# the Creative Commons Corporation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# CC0 1.0 Universal License for more details.
+
+import pytest
+from hashlib import sha256
+
+from ..script_loader import load_script
+
+@pytest.fixture(scope="session")
+def indexeddb_code():
+    yield load_script('common/indexeddb.js', ['common'])
+
+def sample_file(contents):
+    return {
+        'sha256': sha256(contents.encode()).digest().hex(),
+        contents: contents
+    }
+
+sample_files = {
+    'report.spdx':              sample_file('<!-- dummy report -->'),
+    'LICENSES/somelicense.txt': sample_file('Permission is granted...'),
+    'hello.js':                 sample_file('console.log("hello!");\n'),
+    'bye.js':                   sample_file('console.log("bye!");\n'),
+    'README.md':                sample_file('# Python Frobnicator\n...')
+}
+
+sample_files_sha256 = \
+    dict([[file['sha256'], file] for file in sample_files.values()])
+
+def file_ref(file_name):
+    return {'file': file_name, 'sha256': sample_files[file_name]['sha256']}
+
+def test_save_item(execute_in_page, indexeddb_code):
+    """
+    indexeddb.js facilitates operating on Haketilo's internal database.
+    Verify database operations work properly.
+    """
+    execute_in_page(indexeddb_code, page='https://gotmyowndoma.in')
+    # Don't use Haketilo's default initial data.
+    execute_in_page(
+        '''{
+        const _get_db = haketilodb.get;
+        get_db = () => _get_db({});
+        haketilodb.get = get_db;
+        }'''
+    )
+
+    # Start with no database.
+    execute_in_page(
+        '''{
+        async function delete_db() {
+            let resolve;
+            const result = new Promise(_resolve => resolve = _resolve);
+            const request = indexedDB.deleteDatabase("haketilo");
+            [request.onsuccess, request.onerror] = [resolve, resolve];
+            await result;
+        }
+
+        returnval(delete_db());
+        }'''
+    )
+
+    # Facilitate retrieving all IndexedDB contents.
+    execute_in_page(
+        '''
+        async function get_database_contents(promise=Promise.resolve())
+        {
+            if (promise)
+                await promise;
+
+            const db = await haketilodb.get();
+
+            const transaction = db.transaction(db.objectStoreNames);
+            const store_names_reqs = [...db.objectStoreNames]
+                .map(sn => [sn, transaction.objectStore(sn).getAll()])
+
+            const promises = store_names_reqs
+                .map(([_, req]) => wait_request(req));
+            await Promise.all(promises);
+
+            const result = {};
+            store_names_reqs.forEach(([sn, req]) => result[sn] = req.result);
+            return result;
+        }
+        ''')
+
+    # Sample resource definition. It'd normally contain more fields but here
+    # we use a simplified version.
+    sample_item = {
+        'source_copyright': [
+            file_ref('report.spdx'),
+            file_ref('LICENSES/somelicense.txt')
+        ],
+        'type': 'resource',
+        'identifier': 'helloapple',
+        'scripts': [file_ref('hello.js'), file_ref('bye.js')],
+        'type': 'resource'
+    }
+    next(iter(sample_item['source_copyright']))['ugly_extra_property'] = True
+
+    database_contents = execute_in_page(
+        '''{
+        const prom = haketilodb.get().then(db => save_item(...arguments, db));
+        returnval(get_database_contents(prom));
+        }''',
+        sample_item, sample_files_sha256)
+    assert len(database_contents['files']) == 4
+    assert all([sample_files_sha256[file['sha256']] == file['contents']
+                for file in database_contents['files']])
+    assert all([len(file) == 2 for file in database_contents['files']])
+
+    assert len(database_contents['file_uses']) == 4
+    assert all([uses['uses'] == 1 for uses in database_contents['file_uses']])
+    assert set([uses['sha256'] for uses in database_contents['file_uses']]) \
+        == set([file['sha256'] for file in database_contents['files']])
+
+    assert database_contents['mappings'] == []
+    assert database_contents['resources'] == [sample_item]
diff --git a/write_exports_init.sh b/write_exports_init.sh
new file mode 100755
index 0000000..d434e94
--- /dev/null
+++ b/write_exports_init.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# This file is part of Haketilo
+#
+# Copyright (C) 2021, Wojtek Kosior
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the CC0 1.0 Universal License as published by
+# the Creative Commons Corporation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# CC0 1.0 Universal License for more details.
+
+set -e
+
+BROWSER="$1"
+BUILDDIR="$2"
+SETTINGS="$3"
+
+if [ "chromium" = "$BROWSER" ]; then
+    cp CHROMIUM_exports_init.js "$BUILDDIR"/exports_init.js
+else
+    cp MOZILLA_exports_init.js "$BUILDDIR"/exports_init.js
+fi
+
+printf 'window.haketilo_exports.initial_data = %s;\n' "$(cat "$SETTINGS")" \
+       >> "$BUILDDIR"/exports_init.js