facilitate initialization of IndexedDB for use by Haketilo

2 files changed, 418 insertions, 0 deletions

diff --git a/common/entities.js b/common/entities.js
new file mode 100644
index 0000000..46836b5
--- /dev/null
+++ b/common/entities.js
@@ -0,0 +1,148 @@
+/**
+ * This file is part of Haketilo.
+ *
+ * Function: Operations on resources and mappings.
+ *
+ * Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * As additional permission under GNU GPL version 3 section 7, you
+ * may distribute forms of that code without the copy of the GNU
+ * GPL normally required by section 4, provided you include this
+ * license notice and, in case of non-source distribution, a URL
+ * through which recipients can access the Corresponding Source.
+ * If you modify file(s) with this exception, you may extend this
+ * exception to your version of the file(s), but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * As a special exception to the GPL, any HTML file which merely
+ * makes function calls to this code, and for that purpose
+ * includes it by reference shall be deemed a separate work for
+ * copyright law purposes. If you modify this code, you may extend
+ * this exception to your version of the code, but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * I, Wojtek Kosior, thereby promise not to sue for violation of this file's
+ * license. Although I request that you do not make use this code in a
+ * proprietary program, I am not going to enforce this in court.
+ */
+
+/*
+ * Convert ver_str into an array representation, e.g. for ver_str="4.6.13.0"
+ * return [4, 6, 13, 0].
+ */
+const parse_version = ver_str => ver_str.split(".").map(parseInt);
+
+/*
+ * ver is an array of integers. rev is an optional integer. Produce string
+ * representation of version (optionally with revision number), like:
+ *     1.2.3-5
+ * No version normalization is performed.
+ */
+const version_string = (ver, rev=0) => ver.join(".") + (rev ? `-${rev}` : "");
+
+/* vers should be an array of comparable values. Return the greatest one. */
+const max = vals => Array.reduce(vals, (v1, v2) => v1 > v2 ? v1 : v2);
+
+/*
+ * versioned_item should be a dict with keys being version strings and values
+ * being definitions of the respective versions of a single resource/mapping.
+ * Example:
+ *     {
+ *         "1": {
+ *             version: [1]//,
+ *             // more stuff
+ *         },
+ *         "1.1": {
+ *             version: [1, 1]//,
+ *             // more stuff
+ *         }
+ *      }
+ *
+ * Returns the definition with the highest version.
+ */
+function get_newest_version(versioned_item)
+{
+    const best_ver = max(Object.keys(versioned_item).map(parse_version));
+    return versioned_item[version_string(best_ver)];
+}
+
+/*
+ * item is a definition of a resource or mapping. Yield all file references
+ * (objects with `file` and `sha256` properties) this definition has.
+ */
+function* get_used_files(item)
+{
+    for (const file of item.source_copyright)
+	yield file;
+
+    if (item.type === "resource") {
+	for (const file of item.scripts || [])
+	    yield file;
+    }
+}
+
+const entities = {
+    get_newest: get_newest_version,
+    get_files:  get_used_files
+};
+
+/*
+ * EXPORTS_START
+ * EXPORT entities
+ * EXPORTS_END
+ */
+
+/*
+ * Note: the functions below were overeagerly written and are not used now but
+ * might prove useful to once we add more functionalities and are hence kept...
+ */
+
+/*
+ * Clone recursively all objects. Leave other items (arrays, strings) untouched.
+ */
+function deep_object_copy(object)
+{
+    const orig = {object};
+    const result = {};
+    const to_copy = [[orig, {}]];
+
+    while (to_copy.length > 0) {
+	const [object, copy] = to_copy.pop();
+
+	for (const [key, value] of Object.entries(object)) {
+	    copy[key] = value;
+
+	    if (typeof value === "object" && !Array.isArray(value)) {
+		const value_copy = {};
+		to_copy.push([value, value_copy]);
+		copy[key] = value_copy;
+	    }
+	}
+    }
+
+    return result.orig;
+}
+
+/* helper function for normalize_version() */
+const version_reductor = (acc, n) => [...(n || acc.length ? [n] : []), ...acc];
+/*
+ * ver is an array of integers. Strip right-most zeroes from ver.
+ *
+ * Returns a *new* array. Doesn't modify its argument.
+ */
+const normalize_version = ver => Array.reduceRight(ver, version_reductor, []);
diff --git a/common/indexeddb.js b/common/indexeddb.js
new file mode 100644
index 0000000..d0bacc4
--- /dev/null
+++ b/common/indexeddb.js
@@ -0,0 +1,270 @@
+/**
+ * This file is part of Haketilo.
+ *
+ * Function: Facilitate use of IndexedDB within Haketilo.
+ *
+ * Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * As additional permission under GNU GPL version 3 section 7, you
+ * may distribute forms of that code without the copy of the GNU
+ * GPL normally required by section 4, provided you include this
+ * license notice and, in case of non-source distribution, a URL
+ * through which recipients can access the Corresponding Source.
+ * If you modify file(s) with this exception, you may extend this
+ * exception to your version of the file(s), but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * As a special exception to the GPL, any HTML file which merely
+ * makes function calls to this code, and for that purpose
+ * includes it by reference shall be deemed a separate work for
+ * copyright law purposes. If you modify this code, you may extend
+ * this exception to your version of the code, but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * I, Wojtek Kosior, thereby promise not to sue for violation of this file's
+ * license. Although I request that you do not make use this code in a
+ * proprietary program, I am not going to enforce this in court.
+ */
+
+/*
+ * IMPORTS_START
+ * IMPORT initial_data
+ * IMPORT entities
+ * IMPORTS_END
+ */
+
+/* Update when changes are made to database schema. Must have 3 elements */
+const db_version = [1, 0, 0];
+
+const nr_reductor = ([i, s], num) => [i - 1, s + num * 1024 ** i];
+const version_nr = ver => Array.reduce(ver.slice(0, 3), nr_reductor, [2, 0])[1];
+
+const stores = 	[
+    ["files",     {keyPath: "sha256"}],
+    ["file_uses", {keyPath: "sha256"}],
+    ["resources", {keyPath: "identifier"}],
+    ["mappings",  {keyPath: "identifier"}]
+];
+
+let db = null;
+
+/* Generate a Promise that resolves when an IndexedDB request succeeds. */
+async function wait_request(idb_request)
+{
+    let resolve, reject;
+    const waiter = new Promise((...cbs) => [resolve, reject] = cbs);
+    [idb_request.onsuccess, idb_request.onerror] = [resolve, reject];
+    return waiter;
+}
+
+/* asynchronous wrapper for IDBObjectStore's get() method. */
+async function idb_get(transaction, store_name, key)
+{
+    const req = transaction.objectStore(store_name).get(key);
+    return (await wait_request(req)).target.result;
+}
+
+/* asynchronous wrapper for IDBObjectStore's put() method. */
+async function idb_put(transaction, store_name, object)
+{
+    return wait_request(transaction.objectStore(store_name).put(object));
+}
+
+/* asynchronous wrapper for IDBObjectStore's delete() method. */
+async function idb_del(transaction, store_name, key)
+{
+    return wait_request(transaction.objectStore(store_name).delete(key));
+}
+
+/* Open haketilo database, asynchronously return an IDBDatabase object. */
+async function get_db(initialization_data=initial_data)
+{
+    if (db)
+	return db;
+
+    let resolve, reject;
+    const waiter = new Promise((...cbs) => [resolve, reject] = cbs);
+
+    const request = indexedDB.open("haketilo", version_nr(db_version));
+    request.onsuccess       = resolve;
+    request.onerror         = ev => reject("db error: " + ev.target.errorCode);
+    request.onupgradeneeded = resolve;
+
+    const event = await waiter;
+    const opened_db = event.target.result;
+
+    if (event instanceof IDBVersionChangeEvent) {
+	/*
+	 * When we move to a new database schema, we will add upgrade logic
+	 * here.
+	 */
+	if (event.oldVersion > 0)
+	    throw "bad db version: " + event.oldVersion;
+
+	let store;
+	for (const [store_name, key_mode] of stores)
+	    store = opened_db.createObjectStore(store_name, key_mode);
+
+	await new Promise(resolve => store.transaction.oncomplete = resolve);
+
+	save_items(db, initialization_data);
+    }
+
+    db = opened_db;
+
+    return db;
+}
+
+/*
+ * How a sample data argument to the function below might look like:
+ *
+ * data = {
+ *     resources: {
+ *         "resource1": {
+ *             "1": {
+ *                 // some stuff
+ *             },
+ *             "1.1": {
+ *                 // some stuff
+ *             }
+ *         },
+ *         "resource2": {
+ *             "0.4.3": {
+ *                 // some stuff
+ *             }
+ *         },
+ *     },
+ *     mappings: {
+ *         "mapping1": {
+ *             "2": {
+ *                 // some stuff
+ *             }
+ *         },
+ *         "mapping2": {
+ *             "0.1": {
+ *                 // some stuff
+ *             }
+ *         },
+ *     },
+ *     files: {
+ *         "sha256-f9444510dc7403e41049deb133f6892aa6a63c05591b2b59e4ee5b234d7bbd99": "console.log(\"hello\");\n",
+ *         "sha256-b857cd521cc82fff30f0d316deba38b980d66db29a5388eb6004579cf743c6fd": "console.log(\"bye\");"
+ *     }
+ * }
+ */
+async function save_items(db, data)
+{
+    const files = data.files;
+    const resources =
+	  Object.values(data.resources || []).map(entities.get_newest);
+    const mappings =
+	  Object.values(data.mappings || []).map(entities.get_newest);
+
+    resources.concat(mappings).forEach(i => save_item(i, data.files, db));
+}
+
+/* helper function of save_item() */
+async function get_file_uses(transaction, file_uses_sha256, file_ref)
+{
+    let uses = file_uses_sha256[file_ref.sha256];
+    if (uses === undefined) {
+	uses = await idb_get(transaction, "file_uses", file_ref.sha256);
+	if (uses)
+	    [uses.new, uses.initial] = [false, uses.uses];
+	else
+	    uses = {sha256: file_ref.sha256, uses: 0, new: true, initial: 0};
+
+	file_uses_sha256[file_ref.sha256] = uses;
+    }
+
+    return uses;
+}
+
+/*
+ * Save given definition of a resource/mapping to IndexedDB. If the definition
+ * (passed as `item`) references files that are not already present in
+ * IndexedDB, those files should be present as values of the `files_sha256`
+ * object with keys being their sha256 sums.
+ */
+async function save_item(item, files_sha256, db)
+{
+    const store_name = {resource: "resources", mapping: "mappings"}[item.type];
+    const transaction =
+	  db.transaction([store_name, "files", "file_uses"], "readwrite");
+
+    let resolve, reject;
+    const result = new Promise((...cbs) => [resolve, reject] = cbs);
+    transaction.oncomplete = resolve;
+    transaction.onerror = reject;
+
+    const uses_sha256 = {};
+    for (const file_ref of entities.get_files(item))
+	(await get_file_uses(transaction, uses_sha256, file_ref)).uses++;
+
+    const old_item = await idb_get(transaction, store_name, item.identifier);
+    if (old_item !== undefined) {
+	for (const file_ref of entities.get_files(old_item))
+	    (await get_file_uses(transaction, uses_sha256, file_ref)).uses--;
+    }
+
+    for (const uses of Object.values(uses_sha256)) {
+	if (uses.uses < 0)
+	    console.error("internal error: uses < 0 for file " + uses.sha256);
+
+	const [is_new, initial_uses] = [uses.new, uses.initial];
+	delete uses.new;
+	delete uses.initial;
+
+	if (uses.uses < 1) {
+	    if (!is_new) {
+		idb_del(transaction, "file_uses", uses.sha256);
+		idb_del(transaction, "files",     uses.sha256);
+	    }
+
+	    continue;
+	}
+
+	if (uses.uses === initial_uses)
+	    continue;
+
+	const file = files_sha256[uses.sha256];
+	if (file === undefined)
+	    throw "file not present: " + uses.sha256;
+
+	idb_put(transaction, "files", {sha256: uses.sha256, contents: file});
+	idb_put(transaction, "file_uses", uses);
+    }
+
+    idb_put(transaction, store_name, item);
+
+    return result;
+}
+
+const haketilodb = {
+    get:       get_db,
+    save_item: save_item
+};
+
+/*
+ * EXPORTS_START
+ * EXPORT haketilodb
+ * EXPORT idb_get
+ * EXPORT idb_put
+ * EXPORT idb_del
+ * EXPORTS_END
+ */