From 3a90084ec14a15d9b76fa4bfed9e85f15a09dad7 Mon Sep 17 00:00:00 2001 From: Wojtek Kosior Date: Wed, 8 Dec 2021 19:01:50 +0100 Subject: facilitate initialization of IndexedDB for use by Haketilo --- common/entities.js | 148 ++++++++++++++++++++++++++++ common/indexeddb.js | 270 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 418 insertions(+) create mode 100644 common/entities.js create mode 100644 common/indexeddb.js (limited to 'common') diff --git a/common/entities.js b/common/entities.js new file mode 100644 index 0000000..46836b5 --- /dev/null +++ b/common/entities.js @@ -0,0 +1,148 @@ +/** + * This file is part of Haketilo. + * + * Function: Operations on resources and mappings. + * + * Copyright (C) 2021 Wojtek Kosior + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * As additional permission under GNU GPL version 3 section 7, you + * may distribute forms of that code without the copy of the GNU + * GPL normally required by section 4, provided you include this + * license notice and, in case of non-source distribution, a URL + * through which recipients can access the Corresponding Source. + * If you modify file(s) with this exception, you may extend this + * exception to your version of the file(s), but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * As a special exception to the GPL, any HTML file which merely + * makes function calls to this code, and for that purpose + * includes it by reference shall be deemed a separate work for + * copyright law purposes. If you modify this code, you may extend + * this exception to your version of the code, but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * I, Wojtek Kosior, thereby promise not to sue for violation of this file's + * license. Although I request that you do not make use this code in a + * proprietary program, I am not going to enforce this in court. + */ + +/* + * Convert ver_str into an array representation, e.g. for ver_str="4.6.13.0" + * return [4, 6, 13, 0]. + */ +const parse_version = ver_str => ver_str.split(".").map(parseInt); + +/* + * ver is an array of integers. rev is an optional integer. Produce string + * representation of version (optionally with revision number), like: + * 1.2.3-5 + * No version normalization is performed. + */ +const version_string = (ver, rev=0) => ver.join(".") + (rev ? `-${rev}` : ""); + +/* vers should be an array of comparable values. Return the greatest one. */ +const max = vals => Array.reduce(vals, (v1, v2) => v1 > v2 ? v1 : v2); + +/* + * versioned_item should be a dict with keys being version strings and values + * being definitions of the respective versions of a single resource/mapping. + * Example: + * { + * "1": { + * version: [1]//, + * // more stuff + * }, + * "1.1": { + * version: [1, 1]//, + * // more stuff + * } + * } + * + * Returns the definition with the highest version. + */ +function get_newest_version(versioned_item) +{ + const best_ver = max(Object.keys(versioned_item).map(parse_version)); + return versioned_item[version_string(best_ver)]; +} + +/* + * item is a definition of a resource or mapping. Yield all file references + * (objects with `file` and `sha256` properties) this definition has. + */ +function* get_used_files(item) +{ + for (const file of item.source_copyright) + yield file; + + if (item.type === "resource") { + for (const file of item.scripts || []) + yield file; + } +} + +const entities = { + get_newest: get_newest_version, + get_files: get_used_files +}; + +/* + * EXPORTS_START + * EXPORT entities + * EXPORTS_END + */ + +/* + * Note: the functions below were overeagerly written and are not used now but + * might prove useful to once we add more functionalities and are hence kept... + */ + +/* + * Clone recursively all objects. Leave other items (arrays, strings) untouched. + */ +function deep_object_copy(object) +{ + const orig = {object}; + const result = {}; + const to_copy = [[orig, {}]]; + + while (to_copy.length > 0) { + const [object, copy] = to_copy.pop(); + + for (const [key, value] of Object.entries(object)) { + copy[key] = value; + + if (typeof value === "object" && !Array.isArray(value)) { + const value_copy = {}; + to_copy.push([value, value_copy]); + copy[key] = value_copy; + } + } + } + + return result.orig; +} + +/* helper function for normalize_version() */ +const version_reductor = (acc, n) => [...(n || acc.length ? [n] : []), ...acc]; +/* + * ver is an array of integers. Strip right-most zeroes from ver. + * + * Returns a *new* array. Doesn't modify its argument. + */ +const normalize_version = ver => Array.reduceRight(ver, version_reductor, []); diff --git a/common/indexeddb.js b/common/indexeddb.js new file mode 100644 index 0000000..d0bacc4 --- /dev/null +++ b/common/indexeddb.js @@ -0,0 +1,270 @@ +/** + * This file is part of Haketilo. + * + * Function: Facilitate use of IndexedDB within Haketilo. + * + * Copyright (C) 2021 Wojtek Kosior + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * As additional permission under GNU GPL version 3 section 7, you + * may distribute forms of that code without the copy of the GNU + * GPL normally required by section 4, provided you include this + * license notice and, in case of non-source distribution, a URL + * through which recipients can access the Corresponding Source. + * If you modify file(s) with this exception, you may extend this + * exception to your version of the file(s), but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * As a special exception to the GPL, any HTML file which merely + * makes function calls to this code, and for that purpose + * includes it by reference shall be deemed a separate work for + * copyright law purposes. If you modify this code, you may extend + * this exception to your version of the code, but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * I, Wojtek Kosior, thereby promise not to sue for violation of this file's + * license. Although I request that you do not make use this code in a + * proprietary program, I am not going to enforce this in court. + */ + +/* + * IMPORTS_START + * IMPORT initial_data + * IMPORT entities + * IMPORTS_END + */ + +/* Update when changes are made to database schema. Must have 3 elements */ +const db_version = [1, 0, 0]; + +const nr_reductor = ([i, s], num) => [i - 1, s + num * 1024 ** i]; +const version_nr = ver => Array.reduce(ver.slice(0, 3), nr_reductor, [2, 0])[1]; + +const stores = [ + ["files", {keyPath: "sha256"}], + ["file_uses", {keyPath: "sha256"}], + ["resources", {keyPath: "identifier"}], + ["mappings", {keyPath: "identifier"}] +]; + +let db = null; + +/* Generate a Promise that resolves when an IndexedDB request succeeds. */ +async function wait_request(idb_request) +{ + let resolve, reject; + const waiter = new Promise((...cbs) => [resolve, reject] = cbs); + [idb_request.onsuccess, idb_request.onerror] = [resolve, reject]; + return waiter; +} + +/* asynchronous wrapper for IDBObjectStore's get() method. */ +async function idb_get(transaction, store_name, key) +{ + const req = transaction.objectStore(store_name).get(key); + return (await wait_request(req)).target.result; +} + +/* asynchronous wrapper for IDBObjectStore's put() method. */ +async function idb_put(transaction, store_name, object) +{ + return wait_request(transaction.objectStore(store_name).put(object)); +} + +/* asynchronous wrapper for IDBObjectStore's delete() method. */ +async function idb_del(transaction, store_name, key) +{ + return wait_request(transaction.objectStore(store_name).delete(key)); +} + +/* Open haketilo database, asynchronously return an IDBDatabase object. */ +async function get_db(initialization_data=initial_data) +{ + if (db) + return db; + + let resolve, reject; + const waiter = new Promise((...cbs) => [resolve, reject] = cbs); + + const request = indexedDB.open("haketilo", version_nr(db_version)); + request.onsuccess = resolve; + request.onerror = ev => reject("db error: " + ev.target.errorCode); + request.onupgradeneeded = resolve; + + const event = await waiter; + const opened_db = event.target.result; + + if (event instanceof IDBVersionChangeEvent) { + /* + * When we move to a new database schema, we will add upgrade logic + * here. + */ + if (event.oldVersion > 0) + throw "bad db version: " + event.oldVersion; + + let store; + for (const [store_name, key_mode] of stores) + store = opened_db.createObjectStore(store_name, key_mode); + + await new Promise(resolve => store.transaction.oncomplete = resolve); + + save_items(db, initialization_data); + } + + db = opened_db; + + return db; +} + +/* + * How a sample data argument to the function below might look like: + * + * data = { + * resources: { + * "resource1": { + * "1": { + * // some stuff + * }, + * "1.1": { + * // some stuff + * } + * }, + * "resource2": { + * "0.4.3": { + * // some stuff + * } + * }, + * }, + * mappings: { + * "mapping1": { + * "2": { + * // some stuff + * } + * }, + * "mapping2": { + * "0.1": { + * // some stuff + * } + * }, + * }, + * files: { + * "sha256-f9444510dc7403e41049deb133f6892aa6a63c05591b2b59e4ee5b234d7bbd99": "console.log(\"hello\");\n", + * "sha256-b857cd521cc82fff30f0d316deba38b980d66db29a5388eb6004579cf743c6fd": "console.log(\"bye\");" + * } + * } + */ +async function save_items(db, data) +{ + const files = data.files; + const resources = + Object.values(data.resources || []).map(entities.get_newest); + const mappings = + Object.values(data.mappings || []).map(entities.get_newest); + + resources.concat(mappings).forEach(i => save_item(i, data.files, db)); +} + +/* helper function of save_item() */ +async function get_file_uses(transaction, file_uses_sha256, file_ref) +{ + let uses = file_uses_sha256[file_ref.sha256]; + if (uses === undefined) { + uses = await idb_get(transaction, "file_uses", file_ref.sha256); + if (uses) + [uses.new, uses.initial] = [false, uses.uses]; + else + uses = {sha256: file_ref.sha256, uses: 0, new: true, initial: 0}; + + file_uses_sha256[file_ref.sha256] = uses; + } + + return uses; +} + +/* + * Save given definition of a resource/mapping to IndexedDB. If the definition + * (passed as `item`) references files that are not already present in + * IndexedDB, those files should be present as values of the `files_sha256` + * object with keys being their sha256 sums. + */ +async function save_item(item, files_sha256, db) +{ + const store_name = {resource: "resources", mapping: "mappings"}[item.type]; + const transaction = + db.transaction([store_name, "files", "file_uses"], "readwrite"); + + let resolve, reject; + const result = new Promise((...cbs) => [resolve, reject] = cbs); + transaction.oncomplete = resolve; + transaction.onerror = reject; + + const uses_sha256 = {}; + for (const file_ref of entities.get_files(item)) + (await get_file_uses(transaction, uses_sha256, file_ref)).uses++; + + const old_item = await idb_get(transaction, store_name, item.identifier); + if (old_item !== undefined) { + for (const file_ref of entities.get_files(old_item)) + (await get_file_uses(transaction, uses_sha256, file_ref)).uses--; + } + + for (const uses of Object.values(uses_sha256)) { + if (uses.uses < 0) + console.error("internal error: uses < 0 for file " + uses.sha256); + + const [is_new, initial_uses] = [uses.new, uses.initial]; + delete uses.new; + delete uses.initial; + + if (uses.uses < 1) { + if (!is_new) { + idb_del(transaction, "file_uses", uses.sha256); + idb_del(transaction, "files", uses.sha256); + } + + continue; + } + + if (uses.uses === initial_uses) + continue; + + const file = files_sha256[uses.sha256]; + if (file === undefined) + throw "file not present: " + uses.sha256; + + idb_put(transaction, "files", {sha256: uses.sha256, contents: file}); + idb_put(transaction, "file_uses", uses); + } + + idb_put(transaction, store_name, item); + + return result; +} + +const haketilodb = { + get: get_db, + save_item: save_item +}; + +/* + * EXPORTS_START + * EXPORT haketilodb + * EXPORT idb_get + * EXPORT idb_put + * EXPORT idb_del + * EXPORTS_END + */ -- cgit v1.2.3