From 7218849ae2f43aee6b3462a30e07caf5bac3d22b Mon Sep 17 00:00:00 2001 From: Wojtek Kosior Date: Sat, 22 Jan 2022 13:49:40 +0100 Subject: add a mapping/resources installation dialog --- background/CORS_bypass_server.js | 2 +- background/patterns_query_manager.js | 2 +- common/entities.js | 18 ++ common/indexeddb.js | 27 +- common/misc.js | 31 ++- html/base.css | 7 + html/dialog.js | 14 +- html/install.html | 113 ++++++++ html/install.js | 453 +++++++++++++++++++++++++++++++ html/item_list.js | 7 +- html/item_preview.js | 15 +- html/payload_create.js | 7 +- html/settings.html | 5 +- html/text_entry_list.js | 10 +- test/unit/test_CORS_bypass_server.py | 2 +- test/unit/test_indexeddb.py | 38 +-- test/unit/test_install.py | 429 +++++++++++++++++++++++++++++ test/unit/test_patterns_query_manager.py | 4 +- test/unit/test_payload_create.py | 12 +- test/unit/test_repo_query_cacher.py | 33 +-- test/unit/utils.py | 42 +++ test/world_wide_library.py | 91 ++++++- 22 files changed, 1250 insertions(+), 112 deletions(-) create mode 100644 html/install.html create mode 100644 html/install.js create mode 100644 test/unit/test_install.py diff --git a/background/CORS_bypass_server.js b/background/CORS_bypass_server.js index 664100b..9805d41 100644 --- a/background/CORS_bypass_server.js +++ b/background/CORS_bypass_server.js @@ -48,7 +48,7 @@ async function get_prop(object, prop, result_object, call_prop=false) { try { result_object[prop] = call_prop ? (await object[prop]()) : object[prop]; } catch(e) { - result_object[`error-${prop}`] = "" + e; + result_object[`error_${prop}`] = "" + e; } } diff --git a/background/patterns_query_manager.js b/background/patterns_query_manager.js index 8b563ef..78cd0ef 100644 --- a/background/patterns_query_manager.js +++ b/background/patterns_query_manager.js @@ -144,7 +144,7 @@ async function start(secret_) secret = secret_; const [mapping_tracking, initial_mappings] = - await haketilodb.track.mappings(ch => changed("mappings", ch)); + await haketilodb.track.mapping(ch => changed("mappings", ch)); const [blocking_tracking, initial_blocking] = await haketilodb.track.blocking(ch => changed("blocking", ch)); diff --git a/common/entities.js b/common/entities.js index 60a7e2d..b70661f 100644 --- a/common/entities.js +++ b/common/entities.js @@ -54,6 +54,24 @@ const parse_version = ver_str => ver_str.split(".").map(n => parseInt(n)); * No version normalization is performed. */ const version_string = (ver, rev=0) => ver.join(".") + (rev ? `-${rev}` : ""); +#EXPORT version_string + +/* + * This function overloads on the number of arguments. If one argument is + * passed, it is an item definition (it need not be complete, only identifier, + * version and, if applicable, revision properties are relevant). If two or + * three arguments are given, they are in order: item identifier, item version + * and item revision. + * Returned is a string identifying this version of item. + */ +function item_id_string(...args) { + let def = args[0] + if (args.length > 1) + def = {identifier: args[0], version: args[1], revision: args[2]}; + return !Array.isArray(def.version) ? def.identifier : + `${def.identifier}-${version_string(def.version, def.revision)}`; +} +#EXPORT item_id_string /* vers should be an array of comparable values. Return the greatest one. */ const max = vals => Array.reduce(vals, (v1, v2) => v1 > v2 ? v1 : v2); diff --git a/common/indexeddb.js b/common/indexeddb.js index a18c9be..1b8e574 100644 --- a/common/indexeddb.js +++ b/common/indexeddb.js @@ -61,8 +61,8 @@ const version_nr = ver => Array.reduce(ver.slice(0, 3), nr_reductor, [2, 0])[1]; const stores = [ ["files", {keyPath: "hash_key"}], ["file_uses", {keyPath: "hash_key"}], - ["resources", {keyPath: "identifier"}], - ["mappings", {keyPath: "identifier"}], + ["resource", {keyPath: "identifier"}], + ["mapping", {keyPath: "identifier"}], ["settings", {keyPath: "name"}], ["blocking", {keyPath: "pattern"}], ["repos", {keyPath: "url"}] @@ -175,8 +175,8 @@ function make_context(transaction, files) } /* - * item_store_names should be an array with either string "mappings", string - * "resources" or both. files should be a dict with values being contents of + * item_store_names should be an array with either string "mapping", string + * "resource" or both. files should be an object with values being contents of * files that are to be possibly saved in this transaction and keys of the form * `sha256-`. * @@ -292,7 +292,7 @@ async function finalize_transaction(context) */ async function save_items(data) { - const item_store_names = ["resources", "mappings"]; + const item_store_names = ["resource", "mapping"]; const context = await start_items_transaction(item_store_names, data.files); return _save_items(data.resources, data.mappings, context); @@ -323,15 +323,13 @@ async function _save_items(resources, mappings, context) */ async function save_item(item, context) { - const store_name = {resource: "resources", mapping: "mappings"}[item.type]; - for (const file_ref of entities.get_files(item)) await incr_file_uses(context, file_ref); - broadcast.prepare(context.sender, `idb_changes_${store_name}`, + broadcast.prepare(context.sender, `idb_changes_${item.type}`, item.identifier); - await _remove_item(store_name, item.identifier, context, false); - await idb_put(context.transaction, store_name, item); + await _remove_item(item.type, item.identifier, context, false); + await idb_put(context.transaction, item.type, item); } #EXPORT save_item @@ -360,10 +358,10 @@ async function remove_item(store_name, identifier, context) await idb_del(context.transaction, store_name, identifier); } -const remove_resource = (id, ctx) => remove_item("resources", id, ctx); +const remove_resource = (id, ctx) => remove_item("resource", id, ctx); #EXPORT remove_resource -const remove_mapping = (id, ctx) => remove_item("mappings", id, ctx); +const remove_mapping = (id, ctx) => remove_item("mapping", id, ctx); #EXPORT remove_mapping /* Function to retrieve all items from a given store. */ @@ -460,7 +458,8 @@ async function track_change(tracking, key) /* * Monitor changes to `store_name` IndexedDB object store. * - * `store_name` should be either "resources", "mappings" or "settings". + * `store_name` should be either "resource", "mapping", "settings", "blocking" + * or "repos". * * `onchange` should be a callback that will be called when an item is added, * modified or removed from the store. The callback will be passed an object @@ -491,7 +490,7 @@ async function start_tracking(store_name, onchange) } const track = {}; -const trackable = ["resources", "mappings", "settings", "blocking", "repos"]; +const trackable = ["resource", "mapping", "settings", "blocking", "repos"]; for (const store_name of trackable) track[store_name] = onchange => start_tracking(store_name, onchange); #EXPORT track diff --git a/common/misc.js b/common/misc.js index ba14a33..ed8f400 100644 --- a/common/misc.js +++ b/common/misc.js @@ -45,25 +45,30 @@ #FROM common/browser.js IMPORT browser #FROM common/stored_types.js IMPORT TYPE_NAME, TYPE_PREFIX +/* uint8_to_hex is a separate function used in cryptographic functions. */ +const uint8_to_hex = + array => [...array].map(b => ("0" + b.toString(16)).slice(-2)).join(""); + /* - * generating unique, per-site value that can be computed synchronously - * and is impossible to guess for a malicious website + * Asynchronously compute hex string representation of a sha256 digest of a + * UTF-8 string. */ - -/* Uint8toHex is a separate function not exported as (a) it's useful and (b) it will be used in crypto.subtle-based digests */ -function Uint8toHex(data) -{ - let returnValue = ''; - for (let byte of data) - returnValue += ('00' + byte.toString(16)).slice(-2); - return returnValue; +async function sha256_async(string) { + const input_ab = new TextEncoder("utf-8").encode(string); + const digest_ab = await crypto.subtle.digest("SHA-256", input_ab); + return uint8_to_hex(new Uint8Array(digest_ab)); } +#EXPORT sha256_async +/* + * Generate a unique value that can be computed synchronously and is impossible + * to guess for a malicious website. + */ function gen_nonce(length=16) { - let randomData = new Uint8Array(length); - crypto.getRandomValues(randomData); - return Uint8toHex(randomData); + const random_data = new Uint8Array(length); + crypto.getRandomValues(random_data); + return uint8_to_hex(random_data); } #EXPORT gen_nonce diff --git a/html/base.css b/html/base.css index 4575b10..df6213a 100644 --- a/html/base.css +++ b/html/base.css @@ -96,3 +96,10 @@ div.top_line { height: var(--line-height); background: linear-gradient(transparent, #555); } + +.text_center { + text-align: center; +} +.text_right { + text-align: right; +} diff --git a/html/dialog.js b/html/dialog.js index a2406e8..a4e275f 100644 --- a/html/dialog.js +++ b/html/dialog.js @@ -131,11 +131,15 @@ const loader = (ctx, ...msg) => show_dialog(ctx, null, msg); #EXPORT loader /* - * Wrapper around target.addEventListener() that makes the requested callback - * only execute if dialog is not shown. + * Wrapper the requested callback into one that only executes it if dialog is + * not shown. */ -function onevent(ctx, target, event, cb) +function when_hidden(ctx, cb) { - target.addEventListener(event, e => !ctx.shown && cb(e)); + function wrapped_cb(...args) { + if (!ctx.shown) + return cb(...args); + } + return wrapped_cb; } -#EXPORT onevent +#EXPORT when_hidden diff --git a/html/install.html b/html/install.html new file mode 100644 index 0000000..b8d0927 --- /dev/null +++ b/html/install.html @@ -0,0 +1,113 @@ +#IF !INSTALL_LOADED +#DEFINE INSTALL_LOADED + + + + +#INCLUDE html/dialog.html +#INCLUDE html/item_preview.html + +#LOADCSS html/reset.css +#LOADCSS html/base.css + + +#ENDIF diff --git a/html/install.js b/html/install.js new file mode 100644 index 0000000..dbc490d --- /dev/null +++ b/html/install.js @@ -0,0 +1,453 @@ +/** + * This file is part of Haketilo. + * + * Function: Install mappings/resources in Haketilo. + * + * Copyright (C) 2022 Wojtek Kosior + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * As additional permission under GNU GPL version 3 section 7, you + * may distribute forms of that code without the copy of the GNU + * GPL normally required by section 4, provided you include this + * license notice and, in case of non-source distribution, a URL + * through which recipients can access the Corresponding Source. + * If you modify file(s) with this exception, you may extend this + * exception to your version of the file(s), but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * As a special exception to the GPL, any HTML file which merely + * makes function calls to this code, and for that purpose + * includes it by reference shall be deemed a separate work for + * copyright law purposes. If you modify this code, you may extend + * this exception to your version of the code, but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * I, Wojtek Kosior, thereby promise not to sue for violation of this file's + * license. Although I request that you do not make use of this code in a + * proprietary program, I am not going to enforce this in court. + */ + +#IMPORT common/indexeddb.js AS haketilodb +#IMPORT html/dialog.js +#IMPORT html/item_preview.js AS ip + +#FROM common/browser.js IMPORT browser +#FROM html/DOM_helpers.js IMPORT clone_template +#FROM common/entities.js IMPORT item_id_string, version_string, get_files +#FROM common/misc.js IMPORT sha256_async AS sha256 + +const coll = new Intl.Collator(); + +/* + * Comparator used to sort items in the order we want them to appear in + * install dialog: first mappings alphabetically, then resources alphabetically. + */ +function compare_items(def1, def2) { + if (def1.type !== def2.type) + return def1.type === "mapping" ? -1 : 1; + + const name_comparison = coll.compare(def1.long_name, def2.long_name); + return name_comparison === 0 ? + coll.compare(def1.identifier, def2.identifier) : name_comparison; +} + +function ItemEntry(install_view, item) { + Object.assign(this, clone_template("install_list_entry")); + this.item_def = item.def; + + this.item_name.innerText = item.def.long_name; + this.item_id.innerText = item_id_string(item.def); + if (item.db_def) { + this.old_ver.innerText = + version_string(item.db_def.version, item.db_def.revision); + this.update_info.classList.remove("hide"); + } + + let preview_cb = () => install_view.preview_item(item.def); + preview_cb = dialog.when_hidden(install_view.dialog_ctx, preview_cb); + this.details_but.addEventListener("click", preview_cb); +} + +const container_ids = [ + "install_preview", + "dialog_container", + "mapping_preview_container", + "resource_preview_container" +]; + +/* + * Work object is used to communicate between asynchronously executing + * functions when computing dependencies tree of an item and when fetching + * files for installation. + */ +async function init_work() { + const work = { + waiting: 0, + is_ok: true, + db: (await haketilodb.get()), + result: [] + }; + + work.err = function (error, user_message) { + if (error) + console.error(error); + work.is_ok = false; + work.reject_cb(user_message); + } + + return [work, + new Promise((...cbs) => [work.resolve_cb, work.reject_cb] = cbs)]; +} + +function InstallView(tab_id, on_view_show, on_view_hide) { + Object.assign(this, clone_template("install_view")); + this.shown = false; + + const show_container = name => { + for (const cid of container_ids) { + if (cid !== name) + this[cid].classList.add("hide"); + } + this[name].classList.remove("hide"); + } + + this.dialog_ctx = dialog.make(() => show_container("dialog_container"), + () => show_container("install_preview")); + this.dialog_container.prepend(this.dialog_ctx.main_div); + + /* Make a link to view a file from the repository. */ + const make_file_link = (preview_ctx, file_ref) => { + const a = document.createElement("a"); + a.href = `${this.repo_url}file/${file_ref.hash_key}`; + a.innerText = file_ref.file; + + return a; + } + + this.previews_ctx = {}; + + this.preview_item = item_def => { + if (!this.shown) + return; + + const fun = ip[`${item_def.type}_preview`]; + const preview_ctx = fun(item_def, this.previews_ctx[item_def.type], + make_file_link); + this.previews_ctx[item_def.type] = preview_ctx; + + const container_name = `${item_def.type}_preview_container`; + show_container(container_name); + this[container_name].prepend(preview_ctx.main_div); + } + + const back_cb = dialog.when_hidden(this.dialog_ctx, + () => show_container("install_preview")); + for (const type of ["resource", "mapping"]) + this[`${type}_back_but`].addEventListener("click", back_cb); + + const process_item = async (work, item_type, id, ver) => { + if (!work.is_ok || work.processed_by_type[item_type].has(id)) + return; + + work.processed_by_type[item_type].add(id); + work.waiting++; + + const url = ver ? + `${this.repo_url}${item_type}/${id}/${ver.join(".")}.json` : + `${this.repo_url}${item_type}/${id}.json`; + const response = + await browser.tabs.sendMessage(tab_id, ["repo_query", url]); + if (!work.is_ok) + return; + + if ("error" in response) { + return work.err(response.error, + "Failure to communicate with repository :("); + } + + if (!response.ok) { + return work.err(null, + `Repository sent HTTP code ${response.status} :(`); + } + + if ("error_json" in response) { + return work.err(response.error_json, + "Repository's response is not valid JSON :("); + } + + if (response.json.api_schema_version > [1]) { + let api_ver = ""; + try { + api_ver = + ` (${version_string(response.json.api_schema_version)})`; + } catch(e) { + console.warn(e); + } + + const captype = item_type[0].toUpperCase() + item_type.substring(1); + const msg = `${captype} ${item_id_string(id, ver)} was served using unsupported Hydrilla API version${api_ver}. You might need to update Haketilo.`; + return work.err(null, msg); + } + + /* TODO: JSON schema validation should be added here. */ + + delete response.json.api_schema_version; + delete response.json.api_schema_revision; + + const files = response.json.source_copyright + .concat(item_type === "resource" ? response.json.scripts : []); + for (const file of files) { + file.hash_key = `sha256-${file.sha256}`; + delete file.sha256; + } + + if (item_type === "mapping") { + for (const res_ref of Object.values(response.json.payloads)) + process_item(work, "resource", res_ref.identifier); + } else { + for (const res_id of (response.json.dependencies || [])) + process_item(work, "resource", res_id); + } + + /* + * At this point we already have JSON definition of the item and we + * triggered processing of its dependencies. We now have to verify if + * the same or newer version of the item is already present in the + * database and if so - omit this item. + */ + const transaction = work.db.transaction(item_type); + try { + var db_def = await haketilodb.idb_get(transaction, item_type, id); + if (!work.is_ok) + return; + } catch(e) { + const msg = "Error accessing Haketilo's internal database :("; + return work.err(e, msg); + } + if (!db_def || db_def.version < response.json.version) + work.result.push({def: response.json, db_def}); + + if (--work.waiting === 0) + work.resolve_cb(work.result); + } + + async function compute_deps(item_type, item_id, item_ver) { + const [work, work_prom] = await init_work(); + work.processed_by_type = {"mapping" : new Set(), "resource": new Set()}; + + process_item(work, item_type, item_id, item_ver); + + const items = await work_prom; + items.sort((i1, i2) => compare_items(i1.def, i2.def)); + return items; + } + + this.show = async (repo_url, item_type, item_id, item_ver) => { + if (this.shown) + return; + + this.shown = true; + + this.repo_url = repo_url; + + dialog.loader(this.dialog_ctx, "Fetching data from repository..."); + + try { + on_view_show(); + } catch(e) { + console.error(e); + } + + try { + var items = await compute_deps(item_type, item_id, item_ver); + } catch(e) { + var dialog_prom = dialog.error(this.dialog_ctx, e); + } + + if (!dialog_prom && items.length === 0) { + const msg = "Nothing to do - packages already installed."; + var dialog_prom = dialog.info(this.dialog_ctx, msg); + } + + if (dialog_prom) { + dialog.close(this.dialog_ctx); + + await dialog_prom; + + hide(); + return; + } + + this.item_entries = items.map(i => new ItemEntry(this, i)); + this.to_install_list.append(...this.item_entries.map(ie => ie.main_li)); + + dialog.close(this.dialog_ctx); + } + + const process_file = async (work, hash_key) => { + if (!work.is_ok) + return; + + work.waiting++; + + try { + var file_uses = await haketilodb.idb_get(work.file_uses_transaction, + "file_uses", hash_key); + if (!work.is_ok) + return; + } catch(e) { + const msg = "Error accessing Haketilo's internal database :("; + return work.err(e, msg); + } + + if (!file_uses) { + const url = `${this.repo_url}file/${hash_key}`; + + try { + var response = await fetch(url); + if (!work.is_ok) + return; + } catch(e) { + const msg = "Failure to communicate with repository :("; + return work.err(e, msg); + } + + if (!response.ok) { + const msg = `Repository sent HTTP code ${response.status} :(`; + return work.err(null, msg); + } + + try { + var text = await response.text(); + if (!work.is_ok) + return; + } catch(e) { + const msg = "Repository's response is not valid text :("; + return work.err(e, msg); + } + + const digest = await sha256(text); + if (!work.is_ok) + return; + if (`sha256-${digest}` !== hash_key) { + const msg = `${url} served a file with different SHA256 cryptographic sum :(`; + return work.err(null, msg); + } + + work.result.push([hash_key, text]); + } + + if (--work.waiting === 0) + work.resolve_cb(work.result); + } + + const get_missing_files = async item_defs => { + const [work, work_prom] = await init_work(); + work.file_uses_transaction = work.db.transaction("file_uses"); + + const processed_files = new Set(); + + for (const item_def of item_defs) { + for (const file of get_files(item_def)) { + if (!processed_files.has(file.hash_key)) { + processed_files.add(file.hash_key); + process_file(work, file.hash_key); + } + } + } + + return processed_files.size > 0 ? work_prom : []; + } + + const perform_install = async () => { + if (!this.show || !this.item_entries) + return; + + dialog.loader(this.dialog_ctx, "Installing..."); + + const item_defs = this.item_entries.map(ie => ie.item_def); + + try { + var files = (await get_missing_files(item_defs)) + .reduce((ac, [hk, txt]) => Object.assign(ac, {[hk]: txt}), {}); + } catch(e) { + var dialog_prom = dialog.error(this.dialog_ctx, e); + } + + if (files !== undefined) { + const data = {files}; + const names = [["mappings", "mapping"], ["resources", "resource"]]; + + for (const [set_name, type] of names) { + const set = {}; + + for (const def of item_defs.filter(def => def.type === type)) + set[def.identifier] = {[version_string(def.version)]: def}; + + data[set_name] = set; + } + + try { + await haketilodb.save_items(data); + } catch(e) { + console.error(e); + const msg = "Error writing to Haketilo's internal database :("; + var dialog_prom = dialog.error(this.dialog_ctx, msg); + } + } + + if (!dialog_prom) { + const msg = "Successfully installed!"; + var dialog_prom = dialog.info(this.dialog_ctx, msg); + } + + dialog.close(this.dialog_ctx); + + await dialog_prom; + + hide(); + } + + const hide = () => { + if (!this.shown) + return; + + this.shown = false; + delete this.item_entries; + [...this.to_install_list.children].forEach(n => n.remove()); + + try { + on_view_hide(); + } catch(e) { + console.error(e); + } + } + + this.when_hidden = cb => { + const wrapped_cb = (...args) => { + if (!this.shown) + return cb(...args); + } + return wrapped_cb; + } + + const hide_cb = dialog.when_hidden(this.dialog_ctx, hide); + this.cancel_but.addEventListener("click", hide_cb); + + const install_cb = dialog.when_hidden(this.dialog_ctx, perform_install); + this.install_but.addEventListener("click", install_cb); +} diff --git a/html/item_list.js b/html/item_list.js index 51950d5..a616713 100644 --- a/html/item_list.js +++ b/html/item_list.js @@ -200,23 +200,22 @@ function on_dialog_hide(list_ctx) async function remove_single_item(item_type, identifier) { - const store = ({resource: "resources", mapping: "mappings"})[item_type]; const transaction_ctx = - await haketilodb.start_items_transaction([store], {}); + await haketilodb.start_items_transaction([item_type], {}); await haketilodb[`remove_${item_type}`](identifier, transaction_ctx); await haketilodb.finalize_transaction(transaction_ctx); } function resource_list() { - return item_list(resource_preview, haketilodb.track.resources, + return item_list(resource_preview, haketilodb.track.resource, id => remove_single_item("resource", id)); } #EXPORT resource_list function mapping_list() { - return item_list(mapping_preview, haketilodb.track.mappings, + return item_list(mapping_preview, haketilodb.track.mapping, id => remove_single_item("mapping", id)); } #EXPORT mapping_list diff --git a/html/item_preview.js b/html/item_preview.js index 474766c..dccf2d4 100644 --- a/html/item_preview.js +++ b/html/item_preview.js @@ -55,6 +55,7 @@ function populate_list(ul, items) } } +/* Link click handler used in make_file_link(). */ async function file_link_clicked(preview_object, file_ref, event) { event.preventDefault(); @@ -71,6 +72,10 @@ async function file_link_clicked(preview_object, file_ref, event) } } +/* + * The default function to use to create file preview link. Links it creates can + * be used to view files from IndexedDB. + */ function make_file_link(preview_object, file_ref) { const a = document.createElement("a"); @@ -82,7 +87,8 @@ function make_file_link(preview_object, file_ref) return a; } -function resource_preview(resource, preview_object, dialog_context) +function resource_preview(resource, preview_object, dialog_context, + make_link_cb=make_file_link) { if (preview_object === undefined) preview_object = clone_template("resource_preview"); @@ -98,7 +104,7 @@ function resource_preview(resource, preview_object, dialog_context) [...preview_object.dependencies.childNodes].forEach(n => n.remove()); populate_list(preview_object.dependencies, resource.dependencies); - const link_maker = file_ref => make_file_link(preview_object, file_ref); + const link_maker = file_ref => make_link_cb(preview_object, file_ref); [...preview_object.scripts.childNodes].forEach(n => n.remove()); populate_list(preview_object.scripts, resource.scripts.map(link_maker)); @@ -113,7 +119,8 @@ function resource_preview(resource, preview_object, dialog_context) } #EXPORT resource_preview -function mapping_preview(mapping, preview_object, dialog_context) +function mapping_preview(mapping, preview_object, dialog_context, + make_link_cb=make_file_link) { if (preview_object === undefined) preview_object = clone_template("mapping_preview"); @@ -138,7 +145,7 @@ function mapping_preview(mapping, preview_object, dialog_context) } } - const link_maker = file_ref => make_file_link(preview_object, file_ref); + const link_maker = file_ref => make_link_cb(preview_object, file_ref); [...preview_object.copyright.childNodes].forEach(n => n.remove()); populate_list(preview_object.copyright, diff --git a/html/payload_create.js b/html/payload_create.js index 503a461..8828809 100644 --- a/html/payload_create.js +++ b/html/payload_create.js @@ -137,12 +137,11 @@ async function save_payload(saving) { const db = await haketilodb.get(); const tx_starter = haketilodb.start_items_transaction; - const tx_ctx = await tx_starter(["resources", "mappings"], saving.files); + const tx_ctx = await tx_starter(["resource", "mapping"], saving.files); - for (const [type, store_name] of - [["resource", "resources"], ["mapping", "mappings"]]) { + for (const type of ["resource", "mapping"]) { if (!saving[`override_${type}`] && - (await haketilodb.idb_get(tx_ctx.transaction, store_name, + (await haketilodb.idb_get(tx_ctx.transaction, type, saving.identifier))) { saving.ask_override = type; return; diff --git a/html/settings.html b/html/settings.html index df5e751..f318e2c 100644 --- a/html/settings.html +++ b/html/settings.html @@ -92,9 +92,6 @@ #repos_list_container, #repos_dialog_container { padding: 0.8em 0.4em 0.4em 0.4em; } - #default_policy_dialog { - text-align: center; - } #blocking_editable_container, #blocking_dialog_container, #repos_list_container, #repos_dialog_container { max-width: var(--content-max-width); @@ -134,7 +131,7 @@

Allow scripts on

-
+
#INCLUDE html/default_blocking_policy.html
diff --git a/html/text_entry_list.js b/html/text_entry_list.js index ff63c79..8cef08f 100644 --- a/html/text_entry_list.js +++ b/html/text_entry_list.js @@ -162,10 +162,11 @@ function Entry(text, list, entry_idx) { [this.cancel_but, this.make_noneditable], [this.noneditable_view, this.make_editable], ]) - dialog.onevent(list.dialog_ctx, node, "click", cb); + node.addEventListener("click", dialog.when_hidden(list.dialog_ctx, cb)); - dialog.onevent(list.dialog_ctx, this.input, "keypress", - e => (e.key === 'Enter') && enter_hit()); + const enter_cb = e => (e.key === 'Enter') && enter_hit(); + this.input.addEventListener("keypress", + dialog.when_hidden(list.dialog_ctx, enter_cb)); if (entry_idx > 0) { const prev_text = list.shown_texts[entry_idx - 1]; @@ -226,7 +227,8 @@ function TextEntryList(dialog_ctx, destroy_cb, const add_new = () => new Entry(null, this, 0); - dialog.onevent(dialog_ctx, this.new_but, "click", add_new); + this.new_but.addEventListener("click", + dialog.when_hidden(dialog_ctx, add_new)); } async function repo_list(dialog_ctx) { diff --git a/test/unit/test_CORS_bypass_server.py b/test/unit/test_CORS_bypass_server.py index 8b845b9..45e4ebb 100644 --- a/test/unit/test_CORS_bypass_server.py +++ b/test/unit/test_CORS_bypass_server.py @@ -97,7 +97,7 @@ def test_CORS_bypass_server(driver, execute_in_page): assert set(results['invalid'].keys()) == {'error'} assert set(results['nonexistent'].keys()) == \ - {'ok', 'status', 'text', 'error-json'} + {'ok', 'status', 'text', 'error_json'} assert results['nonexistent']['ok'] == False assert results['nonexistent']['status'] == 404 assert results['nonexistent']['text'] == 'Handler for this URL not found.' diff --git a/test/unit/test_indexeddb.py b/test/unit/test_indexeddb.py index 07b620c..aea633b 100644 --- a/test/unit/test_indexeddb.py +++ b/test/unit/test_indexeddb.py @@ -78,7 +78,7 @@ def test_haketilodb_item_modifications(driver, execute_in_page): execute_in_page( '''{ - const promise = start_items_transaction(["resources"], arguments[1]) + const promise = start_items_transaction(["resource"], arguments[1]) .then(ctx => save_item(arguments[0], ctx).then(() => ctx)) .then(finalize_transaction); returnval(promise); @@ -97,8 +97,8 @@ def test_haketilodb_item_modifications(driver, execute_in_page): assert set([uses['hash_key'] for uses in database_contents['file_uses']]) \ == set([file['hash_key'] for file in database_contents['files']]) - assert database_contents['mappings'] == [] - assert database_contents['resources'] == [sample_item] + assert database_contents['mapping'] == [] + assert database_contents['resource'] == [sample_item] # See if trying to add an item without providing all its files ends in an # exception and aborts the transaction as it should. @@ -111,7 +111,7 @@ def test_haketilodb_item_modifications(driver, execute_in_page): async function try_add_item() { const context = - await start_items_transaction(["resources"], args[1]); + await start_items_transaction(["resource"], args[1]); try { await save_item(args[0], context); await finalize_transaction(context); @@ -137,7 +137,7 @@ def test_haketilodb_item_modifications(driver, execute_in_page): sample_item = make_sample_mapping() database_contents = execute_in_page( '''{ - const promise = start_items_transaction(["mappings"], arguments[1]) + const promise = start_items_transaction(["mapping"], arguments[1]) .then(ctx => save_item(arguments[0], ctx).then(() => ctx)) .then(finalize_transaction); returnval(promise); @@ -161,9 +161,9 @@ def test_haketilodb_item_modifications(driver, execute_in_page): assert files == dict([(file['hash_key'], file['contents']) for file in sample_files_list]) - del database_contents['resources'][0]['source_copyright'][0]['extra_prop'] - assert database_contents['resources'] == [make_sample_resource()] - assert database_contents['mappings'] == [sample_item] + del database_contents['resource'][0]['source_copyright'][0]['extra_prop'] + assert database_contents['resource'] == [make_sample_resource()] + assert database_contents['mapping'] == [sample_item] # Try removing the items to get an empty database again. results = [None, None] @@ -172,7 +172,7 @@ def test_haketilodb_item_modifications(driver, execute_in_page): f'''{{ const remover = remove_{item_type}; const promise = - start_items_transaction(["{item_type}s"], {{}}) + start_items_transaction(["{item_type}"], {{}}) .then(ctx => remover('helloapple', ctx).then(() => ctx)) .then(finalize_transaction); returnval(promise); @@ -193,8 +193,8 @@ def test_haketilodb_item_modifications(driver, execute_in_page): assert files == dict([(file['hash_key'], file['contents']) for file in sample_files_list]) - assert results[0]['resources'] == [] - assert results[0]['mappings'] == [sample_item] + assert results[0]['resource'] == [] + assert results[0]['mapping'] == [sample_item] assert results[1] == dict([(key, []) for key in results[0].keys()]) @@ -223,8 +223,8 @@ def test_haketilodb_item_modifications(driver, execute_in_page): execute_in_page('initial_data = arguments[0];', initial_data) database_contents = get_db_contents(execute_in_page) - assert database_contents['resources'] == [sample_resource] - assert database_contents['mappings'] == [sample_mapping] + assert database_contents['resource'] == [sample_resource] + assert database_contents['mapping'] == [sample_mapping] @pytest.mark.get_page('https://gotmyowndoma.in') def test_haketilodb_settings(driver, execute_in_page): @@ -407,8 +407,8 @@ def test_haketilodb_track(driver, execute_in_page, wait_elem_text): }''') assert item_counts == [1 for _ in item_counts] for elem_id, json_value in [ - ('resources_helloapple', sample_resource), - ('mappings_helloapple', sample_mapping), + ('resource_helloapple', sample_resource), + ('mapping_helloapple', sample_mapping), ('settings_option15', {'name': 'option15', 'value': '123'}), ('repos_https://hydril.la', {'url': 'https://hydril.la'}), ('blocking_file:///*', {'pattern': 'file:///*', 'allow': False}) @@ -442,8 +442,8 @@ def test_haketilodb_track(driver, execute_in_page, wait_elem_text): driver.switch_to.window(windows[0]) driver.implicitly_wait(10) for elem_id, json_value in [ - ('resources_helloapple-copy', sample_resource2), - ('mappings_helloapple-copy', sample_mapping2), + ('resource_helloapple-copy', sample_resource2), + ('mapping_helloapple-copy', sample_mapping2), ('settings_option22', {'name': 'option22', 'value': 'abc'}), ('repos_https://hydril2.la', {'url': 'https://hydril2.la'}), ('blocking_ftp://a.bc/', {'pattern': 'ftp://a.bc/', 'allow': True}) @@ -457,7 +457,7 @@ def test_haketilodb_track(driver, execute_in_page, wait_elem_text): '''{ async function change_remove_items() { - const store_names = ["resources", "mappings"]; + const store_names = ["resource", "mapping"]; const ctx = await start_items_transaction(store_names, {}); await remove_resource("helloapple", ctx); await remove_mapping("helloapple-copy", ctx); @@ -470,7 +470,7 @@ def test_haketilodb_track(driver, execute_in_page, wait_elem_text): returnval(change_remove_items()); }''') - removed_ids = ['mappings_helloapple-copy', 'resources_helloapple', + removed_ids = ['mapping_helloapple-copy', 'resource_helloapple', 'repos_https://hydril.la', 'blocking_file:///*'] def condition_items_absent_and_changed(driver): for id in removed_ids: diff --git a/test/unit/test_install.py b/test/unit/test_install.py new file mode 100644 index 0000000..cb8fe36 --- /dev/null +++ b/test/unit/test_install.py @@ -0,0 +1,429 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - item installation dialog +""" + +# This file is part of Haketilo +# +# Copyright (C) 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +import json +from selenium.webdriver.support.ui import WebDriverWait + +from ..extension_crafting import ExtraHTML +from ..script_loader import load_script +from .utils import * + +def content_script(): + script = load_script('content/repo_query_cacher.js') + return f'{script}; {tab_id_asker}; start();' + +def background_script(): + script = load_script('background/broadcast_broker.js', + '#IMPORT background/CORS_bypass_server.js') + return f'{script}; {tab_id_responder}; start(); CORS_bypass_server.start();' + +def setup_view(driver, execute_in_page): + tab_id = run_content_script_in_new_window(driver, 'https://gotmyowndoma.in') + + execute_in_page(load_script('html/install.js')) + container_ids, containers_objects = execute_in_page( + ''' + const cb_calls = []; + const install_view = new InstallView(arguments[0], + () => cb_calls.push("show"), + () => cb_calls.push("hide")); + document.body.append(install_view.main_div); + const ets = () => install_view.item_entries; + const shw = slice => [cb_calls.slice(slice || 0), install_view.shown]; + returnval([container_ids, container_ids.map(cid => install_view[cid])]); + ''', + tab_id) + + containers = dict(zip(container_ids, containers_objects)) + + def assert_container_displayed(container_id): + for cid, cobj in zip(container_ids, containers_objects): + assert (cid == container_id) == cobj.is_displayed() + + return containers, assert_container_displayed + +install_ext_data = { + 'content_script': content_script, + 'background_script': background_script, + 'extra_html': ExtraHTML('html/install.html', {}), + 'navigate_to': 'html/install.html' +} + +@pytest.mark.ext_data(install_ext_data) +@pytest.mark.usefixtures('webextension') +@pytest.mark.parametrize('complex_variant', [False, True]) +def test_install_normal_usage(driver, execute_in_page, complex_variant): + """ + Test of the normal package installation procedure with one mapping and, + depending on parameter, one or many resources. + """ + containers, assert_container_displayed = setup_view(driver, execute_in_page) + + assert execute_in_page('returnval(shw());') == [[], False] + + if complex_variant: + # The resource/mapping others depend on. + root_id = 'abcd-defg-ghij' + root_resource_id = f'resource_{root_id}' + root_mapping_id = f'mapping_{root_id}' + # Those ids are used to check the alphabetical ordering. + resource_ids = [f'resource_{letters}' for letters in ( + 'a', 'abcd', root_id, 'b', 'c', + 'd', 'defg', 'e', 'f', + 'g', 'ghij', 'h', 'i', 'j' + )] + files_count = 9 + else: + root_resource_id = f'resource_a' + root_mapping_id = f'mapping_a' + resource_ids = [root_resource_id] + files_count = 0 + + # Preview the installation of a resource, show resource's details, close + # the details and cancel installation. + execute_in_page('returnval(install_view.show(...arguments));', + 'https://hydril.la/', 'resource', root_resource_id) + + assert execute_in_page('returnval(shw());') == [['show'], True] + assert f'{root_resource_id}-2021.11.11-1'\ + in containers['install_preview'].text + assert_container_displayed('install_preview') + + entries = execute_in_page('returnval(ets().map(e => e.main_li.innerText));') + assert len(entries) == len(resource_ids) + # Verify alphabetical ordering. + assert all([id in text for id, text in zip(resource_ids, entries)]) + + assert not execute_in_page('returnval(ets()[0].old_ver);').is_displayed() + execute_in_page('returnval(ets()[0].details_but);').click() + assert 'resource_a' in containers['resource_preview_container'].text + assert_container_displayed('resource_preview_container') + + execute_in_page('returnval(install_view.resource_back_but);').click() + assert_container_displayed('install_preview') + + assert execute_in_page('returnval(shw());') == [['show'], True] + execute_in_page('returnval(install_view.cancel_but);').click() + assert execute_in_page('returnval(shw());') == [['show', 'hide'], False] + + # Preview the installation of a mapping and a resource, show mapping's + # details, close the details and commit the installation. + execute_in_page('returnval(install_view.show(...arguments));', + 'https://hydril.la/', 'mapping', + root_mapping_id, [2022, 5, 10]) + + assert execute_in_page('returnval(shw(2));') == [['show'], True] + assert_container_displayed('install_preview') + + entries = execute_in_page('returnval(ets().map(e => e.main_li.innerText));') + assert len(entries) == len(resource_ids) + 1 + assert f'{root_mapping_id}-2022.5.10' in entries[0] + # Verify alphabetical ordering. + assert all([id in text for id, text in zip(resource_ids, entries[1:])]) + + assert not execute_in_page('returnval(ets()[0].old_ver);').is_displayed() + execute_in_page('returnval(ets()[0].details_but);').click() + assert root_mapping_id in containers['mapping_preview_container'].text + assert_container_displayed('mapping_preview_container') + + execute_in_page('returnval(install_view.mapping_back_but);').click() + assert_container_displayed('install_preview') + + execute_in_page('returnval(install_view.install_but);').click() + installed = lambda d: 'ly installed!' in containers['dialog_container'].text + WebDriverWait(driver, 10000).until(installed) + + assert execute_in_page('returnval(shw(2));') == [['show'], True] + execute_in_page('returnval(install_view.dialog_ctx.ok_but);').click() + assert execute_in_page('returnval(shw(2));') == [['show', 'hide'], False] + + # Verify the install + db_contents = get_db_contents(execute_in_page) + for item_type, ids in \ + [('mapping', {root_mapping_id}), ('resource', set(resource_ids))]: + assert set([it['identifier'] for it in db_contents[item_type]]) == ids + + assert all([len(db_contents[store]) == files_count + for store in ('files', 'file_uses')]) + + # Update the installed mapping to a newer version. + execute_in_page('returnval(install_view.show(...arguments));', + 'https://hydril.la/', 'mapping', root_mapping_id) + assert execute_in_page('returnval(shw(4));') == [['show'], True] + # resources are already in the newest versions, hence they should not appear + # in the install preview list. + assert execute_in_page('returnval(ets().length);') == 1 + # Mapping's version update information should be displayed. + assert execute_in_page('returnval(ets()[0].old_ver);').is_displayed() + execute_in_page('returnval(install_view.install_but);').click() + + WebDriverWait(driver, 10).until(installed) + + assert execute_in_page('returnval(shw(4));') == [['show'], True] + execute_in_page('returnval(install_view.dialog_ctx.ok_but);').click() + assert execute_in_page('returnval(shw(4));') == [['show', 'hide'], False] + + # Verify the newer version install. + old_db_contents, db_contents = db_contents, get_db_contents(execute_in_page) + old_db_contents['mapping'][0]['version'][-1] += 1 + assert db_contents['mapping'] == old_db_contents['mapping'] + + # All items are up to date - verify dialog is instead shown in this case. + execute_in_page('install_view.show(...arguments);', + 'https://hydril.la/', 'mapping', root_mapping_id) + assert execute_in_page('returnval(shw(6));') == [['show'], True] + assert_container_displayed('dialog_container') + assert 'Nothing to do - packages already installed.' \ + in containers['dialog_container'].text + execute_in_page('returnval(install_view.dialog_ctx.ok_but);').click() + assert execute_in_page('returnval(shw(6));') == [['show', 'hide'], False] + +@pytest.mark.ext_data(install_ext_data) +@pytest.mark.usefixtures('webextension') +@pytest.mark.parametrize('message', [ + 'fetching_data', + 'failure_to_communicate_sendmessage', + 'HTTP_code_item', + 'invalid_JSON', + 'newer_API_version', + 'invalid_API_version', + 'indexeddb_error_item', + 'installing', + 'indexeddb_error_file_uses', + 'failure_to_communicate_fetch', + 'HTTP_code_file', + 'not_valid_text', + 'sha256_mismatch', + 'indexeddb_error_write' +]) +def test_install_dialogs(driver, execute_in_page, message): + """ + Test of various error and loading messages used in install view. + """ + containers, assert_container_displayed = setup_view(driver, execute_in_page) + + def dlg_buts(): + return execute_in_page( + '''{ + const dlg = install_view.dialog_ctx; + const ids = ['ask_buts', 'conf_buts']; + returnval(ids.filter(id => !dlg[id].classList.contains("hide"))); + }''') + + def dialog_txt(): + return execute_in_page( + 'returnval(install_view.dialog_ctx.msg.textContent);' + ) + + def assert_dlg(awaited_buttons, expected_msg, hides_install_view=True, + button_to_click='ok_but'): + WebDriverWait(driver, 10).until(lambda d: dlg_buts() == awaited_buttons) + + assert expected_msg == dialog_txt() + + execute_in_page( + f'returnval(install_view.dialog_ctx.{button_to_click});' + ).click() + + if hides_install_view: + assert execute_in_page('returnval(shw());') == \ + [['show', 'hide'], False] + + if message == 'fetching_data': + execute_in_page( + ''' + browser.tabs.sendMessage = () => new Promise(cb => {}); + install_view.show(...arguments); + ''', + 'https://hydril.la/', 'mapping', 'mapping_a') + + assert dlg_buts() == [] + assert dialog_txt() == 'Fetching data from repository...' + elif message == 'failure_to_communicate_sendmessage': + execute_in_page( + ''' + browser.tabs.sendMessage = () => Promise.resolve({error: "sth"}); + install_view.show(...arguments); + ''', + 'https://hydril.la/', 'mapping', 'mapping_a') + + assert_dlg(['conf_buts'], 'Failure to communicate with repository :(') + elif message == 'HTTP_code_item': + execute_in_page( + ''' + const response = {ok: false, status: 404}; + browser.tabs.sendMessage = () => Promise.resolve(response); + install_view.show(...arguments); + ''', + 'https://hydril.la/', 'mapping', 'mapping_a') + + assert_dlg(['conf_buts'], 'Repository sent HTTP code 404 :(') + elif message == 'invalid_JSON': + execute_in_page( + ''' + const response = {ok: true, status: 200, error_json: "sth"}; + browser.tabs.sendMessage = () => Promise.resolve(response); + install_view.show(...arguments); + ''', + 'https://hydril.la/', 'mapping', 'mapping_a') + + assert_dlg(['conf_buts'], "Repository's response is not valid JSON :(") + elif message == 'newer_API_version': + execute_in_page( + ''' + const response = { + ok: true, + status: 200, + json: {api_schema_version: [99, 99]} + }; + browser.tabs.sendMessage = () => Promise.resolve(response); + install_view.show(...arguments); + ''', + 'https://hydril.la/', 'mapping', 'somemapping', [2, 1]) + + assert_dlg(['conf_buts'], + 'Mapping somemapping-2.1 was served using unsupported Hydrilla API version (99.99). You might need to update Haketilo.') + elif message == 'invalid_API_version': + execute_in_page( + ''' + const response = { + ok: true, + status: 200, + /* API version here is not an array as it should be. */ + json: {api_schema_version: 123} + }; + browser.tabs.sendMessage = () => Promise.resolve(response); + install_view.show(...arguments); + ''', + 'https://hydril.la/', 'resource', 'someresource') + + assert_dlg(['conf_buts'], + 'Resource someresource was served using unsupported Hydrilla API version. You might need to update Haketilo.') + elif message == 'indexeddb_error_item': + execute_in_page( + ''' + haketilodb.idb_get = () => {throw "some error";}; + install_view.show(...arguments); + ''', + 'https://hydril.la/', 'mapping', 'mapping_a') + + assert_dlg(['conf_buts'], + "Error accessing Haketilo's internal database :(") + elif message == 'installing': + execute_in_page( + ''' + haketilodb.save_items = () => new Promise(() => {}); + returnval(install_view.show(...arguments)); + ''', + 'https://hydril.la/', 'mapping', 'mapping_b') + + execute_in_page('returnval(install_view.install_but);').click() + + assert dlg_buts() == [] + assert dialog_txt() == 'Installing...' + elif message == 'indexeddb_error_file_uses': + execute_in_page( + ''' + const old_idb_get = haketilodb.idb_get; + haketilodb.idb_get = function(transaction, store_name, identifier) { + if (store_name === "file_uses") + throw "some error"; + return old_idb_get(...arguments); + } + returnval(install_view.show(...arguments)); + ''', + 'https://hydril.la/', 'mapping', 'mapping_b') + + execute_in_page('returnval(install_view.install_but);').click() + + assert_dlg(['conf_buts'], + "Error accessing Haketilo's internal database :(") + elif message == 'failure_to_communicate_fetch': + execute_in_page( + ''' + fetch = () => {throw "some error";}; + returnval(install_view.show(...arguments)); + ''', + 'https://hydril.la/', 'mapping', 'mapping_b') + + execute_in_page('returnval(install_view.install_but);').click() + + assert_dlg(['conf_buts'], + 'Failure to communicate with repository :(') + elif message == 'HTTP_code_file': + execute_in_page( + ''' + fetch = () => Promise.resolve({ok: false, status: 400}); + returnval(install_view.show(...arguments)); + ''', + 'https://hydril.la/', 'mapping', 'mapping_b') + + execute_in_page('returnval(install_view.install_but);').click() + + assert_dlg(['conf_buts'], 'Repository sent HTTP code 400 :(') + elif message == 'not_valid_text': + execute_in_page( + ''' + const err = () => {throw "some error";}; + fetch = () => Promise.resolve({ok: true, status: 200, text: err}); + returnval(install_view.show(...arguments)); + ''', + 'https://hydril.la/', 'mapping', 'mapping_b') + + execute_in_page('returnval(install_view.install_but);').click() + + assert_dlg(['conf_buts'], "Repository's response is not valid text :(") + elif message == 'sha256_mismatch': + execute_in_page( + ''' + let old_fetch = fetch, url_used; + fetch = async function(url) { + url_used = url; + const response = await old_fetch(...arguments); + const text = () => response.text().then(t => t + ":d"); + return {ok: response.ok, status: response.status, text}; + } + returnval(install_view.show(...arguments)); + ''', + 'https://hydril.la/', 'mapping', 'mapping_b') + + execute_in_page('returnval(install_view.install_but);').click() + + get_url_used = lambda d: execute_in_page('returnval(url_used);') + url_used = WebDriverWait(driver, 10).until(get_url_used) + print ((url_used,)) + + assert dlg_buts() == ['conf_buts'] + assert dialog_txt() == \ + f'{url_used} served a file with different SHA256 cryptographic sum :(' + elif message == 'indexeddb_error_write': + execute_in_page( + ''' + haketilodb.save_items = () => {throw "some error";}; + returnval(install_view.show(...arguments)); + ''', + 'https://hydril.la/', 'mapping', 'mapping_b') + + execute_in_page('returnval(install_view.install_but);').click() + + assert_dlg(['conf_buts'], + "Error writing to Haketilo's internal database :(") + else: + raise Exception('made a typo in test function params?') diff --git a/test/unit/test_patterns_query_manager.py b/test/unit/test_patterns_query_manager.py index 35047f5..4e6d1bf 100644 --- a/test/unit/test_patterns_query_manager.py +++ b/test/unit/test_patterns_query_manager.py @@ -89,7 +89,7 @@ def test_pqm_tree_building(driver, execute_in_page): const [initial_mappings, initial_blocking] = arguments.slice(0, 2); let mappingchange, blockingchange; - haketilodb.track.mappings = function (cb) { + haketilodb.track.mapping = function (cb) { mappingchange = cb; return [{}, initial_mappings]; @@ -253,7 +253,7 @@ def test_pqm_script_injection(driver, execute_in_page): const identifiers = arguments[0]; async function remove_items() { - const ctx = await start_items_transaction(["mappings"], {}); + const ctx = await start_items_transaction(["mapping"], {}); for (const id of identifiers) await remove_mapping(id, ctx); await finalize_transaction(ctx); diff --git a/test/unit/test_payload_create.py b/test/unit/test_payload_create.py index 6a6cf2c..cee3a9b 100644 --- a/test/unit/test_payload_create.py +++ b/test/unit/test_payload_create.py @@ -118,19 +118,19 @@ def test_payload_create_normal_usage(driver, execute_in_page): def assert_db_contents(): db_contents = get_db_contents(execute_in_page) - assert uuidv4_re.match(db_contents['resources'][0]['uuid']) + assert uuidv4_re.match(db_contents['resource'][0]['uuid']) localid = f'local-{form_data["identifier"]}' long_name = form_data['long_name'] or form_data['identifier'] payloads = dict([(pat, {'identifier': localid}) for pat in form_data['patterns'].split('\n') if pat]) - assert db_contents['resources'] == [{ + assert db_contents['resource'] == [{ 'source_name': localid, 'source_copyright': [], 'type': 'resource', 'identifier': localid, - 'uuid': db_contents['resources'][0]['uuid'], + 'uuid': db_contents['resource'][0]['uuid'], 'version': [1], 'description': form_data['description'], 'dependencies': [], @@ -141,13 +141,13 @@ def test_payload_create_normal_usage(driver, execute_in_page): }] }] - assert uuidv4_re.match(db_contents['mappings'][0]['uuid']) - assert db_contents['mappings'] == [{ + assert uuidv4_re.match(db_contents['mapping'][0]['uuid']) + assert db_contents['mapping'] == [{ 'source_name': localid, 'source_copyright': [], 'type': 'mapping', 'identifier': localid, - 'uuid': db_contents['mappings'][0]['uuid'], + 'uuid': db_contents['mapping'][0]['uuid'], 'version': [1], 'description': form_data['description'], 'long_name': long_name, diff --git a/test/unit/test_repo_query_cacher.py b/test/unit/test_repo_query_cacher.py index d5c7396..ee9f0fd 100644 --- a/test/unit/test_repo_query_cacher.py +++ b/test/unit/test_repo_query_cacher.py @@ -22,40 +22,15 @@ import json from selenium.webdriver.support.ui import WebDriverWait from ..script_loader import load_script - -tab_id_responder = ''' -function tell_tab_id(msg, sender, respond_cb) { - if (msg[0] === "learn_tab_id") - respond_cb(sender.tab.id); -} -browser.runtime.onMessage.addListener(tell_tab_id); -''' +from .utils import * def content_script(): - return load_script('content/repo_query_cacher.js') + '''; - start(); - browser.runtime.sendMessage(["learn_tab_id"]) - .then(tid => window.wrappedJSObject.haketilo_tab = tid); - ''' + script = load_script('content/repo_query_cacher.js') + return f'{script}; {tab_id_asker}; start();' def bypass_js(): return load_script('background/CORS_bypass_server.js') + '; start();' -def run_content_script_in_new_window(driver, url): - initial_handle = driver.current_window_handle - handles = driver.window_handles - driver.execute_script('window.open(arguments[0], "_blank");', url) - WebDriverWait(driver, 10).until(lambda d: d.window_handles is not handles) - new_handle = [h for h in driver.window_handles if h not in handles][0] - - driver.switch_to.window(new_handle) - - get_tab_id = lambda d: d.execute_script('return window.haketilo_tab;') - tab_id = WebDriverWait(driver, 10).until(get_tab_id) - - driver.switch_to.window(initial_handle) - return tab_id - def fetch_through_cache(driver, tab_id, url): return driver.execute_script( ''' @@ -90,7 +65,7 @@ def test_repo_query_cacher_normal_use(driver, execute_in_page): for i in range(2): result = fetch_through_cache(driver, tab_id, 'https://nxdoma.in/') - assert set(result.keys()) == {'ok', 'status', 'error-json'} + assert set(result.keys()) == {'ok', 'status', 'error_json'} assert result['ok'] == False assert result['status'] == 404 diff --git a/test/unit/utils.py b/test/unit/utils.py index 4d8766e..90a2ab7 100644 --- a/test/unit/utils.py +++ b/test/unit/utils.py @@ -26,6 +26,7 @@ Various functions and objects that can be reused between unit tests # proprietary program, I am not going to enforce this in court. from hashlib import sha256 +from selenium.webdriver.support.ui import WebDriverWait from ..script_loader import load_script @@ -200,3 +201,44 @@ def are_scripts_allowed(driver, nonce=None): return document.haketilo_scripts_allowed; ''', nonce) + +""" +tab_id_responder is meant to be appended to background script of a test +extension. +""" +tab_id_responder = ''' +function tell_tab_id(msg, sender, respond_cb) { + if (msg[0] === "learn_tab_id") + respond_cb(sender.tab.id); +} +browser.runtime.onMessage.addListener(tell_tab_id); +''' + +""" +tab_id_asker is meant to be appended to content script of a test extension. +""" +tab_id_asker = ''' +browser.runtime.sendMessage(["learn_tab_id"]) + .then(tid => window.wrappedJSObject.haketilo_tab = tid); +''' + +def run_content_script_in_new_window(driver, url): + """ + Expect an extension to be loaded which had tab_id_responder and tab_id_asker + appended to its background and content scripts, respectively. + Open the provided url in a new tab, find its tab id and return it, with + current window changed back to the initial one. + """ + initial_handle = driver.current_window_handle + handles = driver.window_handles + driver.execute_script('window.open(arguments[0], "_blank");', url) + WebDriverWait(driver, 10).until(lambda d: d.window_handles is not handles) + new_handle = [h for h in driver.window_handles if h not in handles][0] + + driver.switch_to.window(new_handle) + + get_tab_id = lambda d: d.execute_script('return window.haketilo_tab;') + tab_id = WebDriverWait(driver, 10).until(get_tab_id) + + driver.switch_to.window(initial_handle) + return tab_id diff --git a/test/world_wide_library.py b/test/world_wide_library.py index 14e3d2f..31864fb 100644 --- a/test/world_wide_library.py +++ b/test/world_wide_library.py @@ -31,9 +31,14 @@ from hashlib import sha256 from pathlib import Path from shutil import rmtree from threading import Lock +from uuid import uuid4 import json from .misc_constants import here +from .unit.utils import * # sample repo data + +# TODO: instead of having the entire catalog defined here, make it possible to +# add catalog items from within individual test files. served_scripts = {} served_scripts_lock = Lock() @@ -100,6 +105,86 @@ def serve_counter(command, get_params, post_params): json.dumps({'counter': request_counter}) ) +# Mock a Hydrilla repository. + +# Mock files in the repository. +sample_contents = [f'Mi povas manĝi vitron, ĝi ne damaĝas min {i}' + for i in range(9)] +sample_hashes = [sha256(c.encode()).digest().hex() for c in sample_contents] + +file_url = lambda hashed: f'https://hydril.la/file/sha256-{hashed}' +file_handler = lambda contents: lambda c, g, p: (200, {}, contents) + +sample_files_catalog = dict([(file_url(h), file_handler(c)) + for h, c in zip(sample_hashes, sample_contents)]) + +# Mock resources and mappings in the repository. +sample_resource_templates = [] + +for deps in [(0, 1, 2, 3), (3, 4, 5, 6), (6, 7, 8, 9)]: + letters = [chr(ord('a') + i) for i in deps] + sample_resource_templates.append({ + 'id_suffix': ''.join(letters), + 'files_count': deps[0], + 'dependencies': [f'resource_{l}' for l in letters] + }) + +suffixes = [srt['id_suffix'] for srt in sample_resource_templates] +sample_resource_templates.append({ + 'id_suffix': '-'.join(suffixes), + 'files_count': 2, + 'dependencies': [f'resource_{suffix}' for suffix in suffixes] +}) + +for i in range(10): + sample_resource_templates.append({ + 'id_suffix': chr(ord('a') + i), + 'files_count': i, + 'dependencies': [] + }) + +sample_resources_catalog = {} +sample_mappings_catalog = {} + +for srt in sample_resource_templates: + resource = make_sample_resource() + resource['api_schema_version'] = [1] + resource['api_schema_revision'] = 1 + resource['identifier'] = f'resource_{srt["id_suffix"]}' + resource['long_name'] = resource['identifier'].upper() + resource['uuid'] = str(uuid4()) + resource['dependencies'] = srt['dependencies'] + resource['source_copyright'] = [] + resource['scripts'] = [] + for i in range(srt['files_count']): + file_ref = {'file': f'file_{i}', 'sha256': sample_hashes[i]} + resource[('source_copyright', 'scripts')[i & 1]].append(file_ref) + + # Keeping it simple - just make one corresponding mapping for each resource. + payloads = {'https://example.com/*': {'identifier': resource['identifier']}} + + mapping = make_sample_mapping() + mapping['api_schema_version'] = [1] + mapping['api_schema_revision'] = 1 + mapping['identifier'] = f'mapping_{srt["id_suffix"]}' + mapping['long_name'] = mapping['identifier'].upper() + mapping['uuid'] = str(uuid4()) + mapping['source_copyright'] = resource['source_copyright'] + mapping['payloads'] = payloads + + make_handler = lambda txt: lambda c, g, p: (200, {}, txt) + + for item, catalog in [ + (resource, sample_resources_catalog), + (mapping, sample_mappings_catalog) + ]: + fmt = f'https://hydril.la/{item["type"]}/{item["identifier"]}%s.json' + # Make 2 versions of each item so that we can test updates. + for i in range(2): + for fmt_arg in ('', '/' + item_version_string(item)): + catalog[fmt % fmt_arg] = make_handler(json.dumps(item)) + item['version'][-1] += 1 + catalog = { 'http://gotmyowndoma.in': (302, {'location': 'http://gotmyowndoma.in/index.html'}, None), @@ -144,5 +229,9 @@ catalog = { 'https://site.with.paylo.ad/': (302, {'location': 'https://site.with.paylo.ad/index.html'}, None), 'https://site.with.paylo.ad/index.html': - (200, {}, here / 'data' / 'pages' / 'gotmyowndomain_https.html') + (200, {}, here / 'data' / 'pages' / 'gotmyowndomain_https.html'), + + **sample_files_catalog, + **sample_resources_catalog, + **sample_mappings_catalog } -- cgit v1.2.3