diff options
79 files changed, 4004 insertions, 1784 deletions
diff --git a/CHROMIUM_exports_init.js b/CHROMIUM_exports_init.js new file mode 100644 index 0000000..0e61d40 --- /dev/null +++ b/CHROMIUM_exports_init.js @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: CC0-1.0 + +window.haketilo_exports = {is_chrome: true, browser: window.chrome}; diff --git a/MOZILLA_exports_init.js b/MOZILLA_exports_init.js new file mode 100644 index 0000000..a1135e8 --- /dev/null +++ b/MOZILLA_exports_init.js @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +/** + * This file is part of Haketilo. + * + * Function: Data structure to query items by URL patterns. + * + * Copyright (C) 2021 Wojtek Kosior + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * As additional permission under GNU GPL version 3 section 7, you + * may distribute forms of that code without the copy of the GNU + * GPL normally required by section 4, provided you include this + * license notice and, in case of non-source distribution, a URL + * through which recipients can access the Corresponding Source. + * If you modify file(s) with this exception, you may extend this + * exception to your version of the file(s), but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * As a special exception to the GPL, any HTML file which merely + * makes function calls to this code, and for that purpose + * includes it by reference shall be deemed a separate work for + * copyright law purposes. If you modify this code, you may extend + * this exception to your version of the code, but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + * + * I, Wojtek Kosior, thereby promise not to sue for violation of this file's + * license. Although I request that you do not make use this code in a + * proprietary program, I am not going to enforce this in court. + */ + +/* Polyfill for IceCat 60. */ +String.prototype.matchAll = String.prototype.matchAll || function(regex) { + if (regex.flags.search("g") === -1) + throw new TypeError("String.prototype.matchAll called with a non-global RegExp argument"); + + for (const matches = [];;) { + if (matches[matches.push(regex.exec(this)) - 1] === null) + return matches.splice(0, matches.length - 1); + } +} + +window.haketilo_exports = {is_mozilla: true, browser: this.browser}; diff --git a/Makefile.in b/Makefile.in new file mode 100644 index 0000000..5a376e1 --- /dev/null +++ b/Makefile.in @@ -0,0 +1,123 @@ +# This file is part of Haketilo +# +# Copyright (C) 2021, jahoti +# Copyright (C) 2021, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +SHELL = /bin/sh +VPATH = <<VPATH>> + +version = <<VERSION>> +extension_files = background/ common/ content/ html/ licenses/ \ + copyright default_settings.json manifest.json + +metafiles = build.sh configure Makefile.in process_html_file.sh README.txt \ + re-generate_icons.sh shell_utils.sh upload_amo.sh write_makefile.sh + +# Configuration goes here + +# The default target: placed up here +default: $(default_target) + +.PHONY: mozilla install-mozilla chromium install-chromium \ + all all-unpacked default unpacked \ + install install-html install-dvi install-pdf install-ps uninstall \ + install-strip clean distclean mostlyclean maintainer-clean TAGS info \ + dvi html pdf ps dist check installcheck installdirs \ + test test-environment + +# core files +icons/haketilo16.png: icons/haketilo.svg + cd "$(srcdir)" && ./re-generate_icons.sh +# Use haketilo16.png as an "index" for all the icon PNGs + +# browser-specific targets +all: mozilla chromium +mozilla: mozilla-build.zip +chromium: chromium-build.zip + +unpacked: $(default_target)-unpacked +all-unpacked: mozilla-unpacked chromium-unpacked +%-unpacked: $(extension_files) icons/haketilo16.png + $(srcdir)/build.sh $* $(srcdir) $(UPDATE_URL) + +install install-strip: $(default_target)-unpacked + cp -R $(default_target)-unpacked \ + "$(DESTDIR)/{6fe13369-88e9-440f-b837-5012fb3bedec}" + +uninstall: + rm -r "$(DESTDIR)/{6fe13369-88e9-440f-b837-5012fb3bedec}" + +%-build.zip: %-unpacked + cd $< && zip -q -r ../$@ * + +test/certs/: + mkdir $@ + +test/certs/%.key: | test/certs/ + openssl genrsa -out $@ 2048 + +test/certs/rootCA.pem: test/certs/rootCA.key + openssl req -x509 -new -nodes -key $< -days 1024 -out $@ \ + -subj "/CN=Haketilo Test" + +test: test/certs/rootCA.pem test/certs/site.key + MOZ_HEADLESS=whatever pytest + +test-environment: test/certs/rootCA.pem test/certs/site.key + python3 -m test + +# helper targets +clean mostlyclean: + rm -rf mozilla-unpacked chromium-unpacked haketilo-$(version) + rm -f mozilla-build.zip chromium-build.zip haketilo-$(version).tar.gz \ + haketilo-$(version).tar + rm -rf test/certs + rm -rf $$(find . -name geckodriver.log) + rm -rf $$(find . -type d -name __pycache__) + +distclean: clean + rm -f Makefile config.status record.conf + +maintainer-clean: distclean + @echo 'This command is intended for maintainers to use; it' + @echo 'deletes files that may need special tools to rebuild.' + rm -f "$(srcdir)"/icons/*.png + +dist: $(extension_files) $(metafiles) icons/haketilo16.png + test -d haketilo-$(version) || mkdir haketilo-$(version) + for file in $(extension_files) $(metafiles) icons/; do \ + cp -R "$(srcdir)"/$$file haketilo-$(version); \ + done + tar cf haketilo-$(version).tar haketilo-$(version) + gzip haketilo-$(version).tar + +# Files for constructing the makefile +Makefile: config.status Makefile.in record.conf + ./config.status + +config.status: write_makefile.sh + cp "$(srcdir)"/write_makefile.sh config.status + +# Unused GNU-specified targets +install-html: +install-dvi: +install-pdf: +install-ps: +TAGS: +info: +dvi: +html: +pdf: +ps: +check: +installcheck: +installdirs: @@ -1,4 +1,4 @@ -# Hachette - Make The Web Great Again! # +# Haketilo - Make The Web Great Again! # This extension's goal is to allow replacing javascript served by websites with scripts specified by user. Something like NoScript and Greasemonkey @@ -9,7 +9,7 @@ Currently, the target browsers for this extension are Ungoogled Chromium and various forks of Firefox (version 60+). This extension is still in an early stage. Also see -`https://git.koszko.org/browser-extension-doc/' for documentation in +`https://hydrillabugs.koszko.org/projects/haketilo/wiki/' for documentation in development. ## Installation ## @@ -28,6 +28,6 @@ various additional licenses and permissions for particular files. ## Contributing ## Get the code from: https://git.koszko.org/browser-extension/ -Come to: https://hachettebugs.koszko.org/projects/hachette +Come to: https://hydrillabugs.koszko.org/projects/haketilo Optionally, write to $(echo a29zemtvQGtvc3prby5vcmcK | base64 -d) diff --git a/background/main.js b/background/main.js index 7c50fd5..358d549 100644 --- a/background/main.js +++ b/background/main.js @@ -1,5 +1,7 @@ /** - * Hachette main background script + * This file is part of Haketilo. + * + * Function: Main background script. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. @@ -9,20 +11,24 @@ * IMPORTS_START * IMPORT TYPE_PREFIX * IMPORT get_storage + * IMPORT light_storage * IMPORT start_storage_server * IMPORT start_page_actions_server - * IMPORT start_policy_injector * IMPORT browser + * IMPORT is_privileged_url + * IMPORT query_best + * IMPORT inject_csp_headers + * IMPORT apply_stream_filter + * IMPORT is_chrome + * IMPORT is_mozilla * IMPORTS_END */ start_storage_server(); start_page_actions_server(); -start_policy_injector(); async function init_ext(install_details) { - console.log("details:", install_details); if (install_details.reason != "install") return; @@ -44,4 +50,156 @@ async function init_ext(install_details) browser.runtime.onInstalled.addListener(init_ext); -console.log("hello, hachette"); +/* + * The function below implements a more practical interface for what it does by + * wrapping the old query_best() function. + */ +function decide_policy_for_url(storage, policy_observable, url) +{ + if (storage === undefined) + return {allow: false}; + + const settings = + {allow: policy_observable !== undefined && policy_observable.value}; + + const [pattern, queried_settings] = query_best(storage, url); + + if (queried_settings) { + settings.payload = queried_settings.components; + settings.allow = !!queried_settings.allow && !settings.payload; + settings.pattern = pattern; + } + + return settings; +} + +let storage; +let policy_observable = {}; + +function sanitize_web_page(details) +{ + const url = details.url; + if (is_privileged_url(details.url)) + return; + + const policy = + decide_policy_for_url(storage, policy_observable, details.url); + + let headers = details.responseHeaders; + + headers = inject_csp_headers(headers, policy); + + let skip = false; + for (const header of headers) { + if ((header.name.toLowerCase().trim() === "content-disposition" && + /^\s*attachment\s*(;.*)$/i.test(header.value))) + skip = true; + } + skip = skip || (details.statusCode >= 300 && details.statusCode < 400); + + if (!skip) { + /* Check for API availability. */ + if (browser.webRequest.filterResponseData) + headers = apply_stream_filter(details, headers, policy); + } + + return {responseHeaders: headers}; +} + +const request_url_regex = /^[^?]*\?url=(.*)$/; +const redirect_url_template = browser.runtime.getURL("dummy") + "?settings="; + +function synchronously_smuggle_policy(details) +{ + /* + * Content script will make a synchronous XmlHttpRequest to extension's + * `dummy` file to query settings for given URL. We smuggle that + * information in query parameter of the URL we redirect to. + * A risk of fingerprinting arises if a page with script execution allowed + * guesses the dummy file URL and makes an AJAX call to it. It is currently + * a problem in ManifestV2 Chromium-family port of Haketilo because Chromium + * uses predictable URLs for web-accessible resources. We plan to fix it in + * the future ManifestV3 port. + */ + if (details.type !== "xmlhttprequest") + return {cancel: true}; + + console.debug(`Settings queried using XHR for '${details.url}'.`); + + let policy = {allow: false}; + + try { + /* + * request_url should be of the following format: + * <url_for_extension's_dummy_file>?url=<valid_urlencoded_url> + */ + const match = request_url_regex.exec(details.url); + const queried_url = decodeURIComponent(match[1]); + + if (details.initiator && !queried_url.startsWith(details.initiator)) { + console.warn(`Blocked suspicious query of '${url}' by '${details.initiator}'. This might be the result of page fingerprinting the browser.`); + return {cancel: true}; + } + + policy = decide_policy_for_url(storage, policy_observable, queried_url); + } catch (e) { + console.warn(`Bad request! Expected ${browser.runtime.getURL("dummy")}?url=<valid_urlencoded_url>. Got ${request_url}. This might be the result of page fingerprinting the browser.`); + } + + const encoded_policy = encodeURIComponent(JSON.stringify(policy)); + + return {redirectUrl: redirect_url_template + encoded_policy}; +} + +const all_types = [ + "main_frame", "sub_frame", "stylesheet", "script", "image", "font", + "object", "xmlhttprequest", "ping", "csp_report", "media", "websocket", + "other", "main_frame", "sub_frame" +]; + +async function start_webRequest_operations() +{ + storage = await get_storage(); + + const extra_opts = ["blocking"]; + if (is_chrome) + extra_opts.push("extraHeaders"); + + browser.webRequest.onHeadersReceived.addListener( + sanitize_web_page, + {urls: ["<all_urls>"], types: ["main_frame", "sub_frame"]}, + extra_opts.concat("responseHeaders") + ); + + const dummy_url_pattern = browser.runtime.getURL("dummy") + "?url=*"; + browser.webRequest.onBeforeRequest.addListener( + synchronously_smuggle_policy, + {urls: [dummy_url_pattern], types: ["xmlhttprequest"]}, + extra_opts + ); + + policy_observable = await light_storage.observe_var("default_allow"); +} + +start_webRequest_operations(); + +const code = `\ +console.warn("Hi, I'm Mr Dynamic!"); + +console.debug("let's see how window.haketilo_exports looks like now"); + +console.log("haketilo_exports", window.haketilo_exports); +` + +async function test_dynamic_content_scripts() +{ + browser.contentScripts.register({ + "js": [{code}], + "matches": ["<all_urls>"], + "allFrames": true, + "runAt": "document_start" +}); +} + +if (is_mozilla) + test_dynamic_content_scripts(); diff --git a/background/page_actions_server.js b/background/page_actions_server.js index 58a0073..74783c9 100644 --- a/background/page_actions_server.js +++ b/background/page_actions_server.js @@ -1,5 +1,7 @@ /** - * Hachette serving of page actions to content scripts + * This file is part of Haketilo. + * + * Function: Serving page actions to content scripts. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. @@ -8,12 +10,12 @@ /* * IMPORTS_START * IMPORT get_storage + * IMPORT light_storage * IMPORT TYPE_PREFIX * IMPORT CONNECTION_TYPE * IMPORT browser * IMPORT listen_for_connection * IMPORT sha256 - * IMPORT query_best * IMPORT make_ajax_request * IMPORTS_END */ @@ -21,23 +23,6 @@ var storage; var handler; -function send_actions(url, port) -{ - const [pattern, settings] = query_best(storage, url); - const repos = storage.get_all(TYPE_PREFIX.REPO); - - port.postMessage(["settings", [pattern, settings, repos]]); - - if (settings === undefined) - return; - - let components = settings.components; - let processed_bags = new Set(); - - if (components !== undefined) - send_scripts([components], port, processed_bags); -} - // TODO: parallelize script fetching async function send_scripts(components, port, processed_bags) { @@ -109,9 +94,11 @@ async function fetch_remote_script(script_data) function handle_message(port, message, handler) { port.onMessage.removeListener(handler[0]); - let url = message.url; - console.log({url}); - send_actions(url, port); + console.debug(`Loading payload '${message.payload}'.`); + + const processed_bags = new Set(); + + send_scripts([message.payload], port, processed_bags); } function new_connection(port) diff --git a/background/policy_injector.js b/background/policy_injector.js index 9725e99..b49ec47 100644 --- a/background/policy_injector.js +++ b/background/policy_injector.js @@ -1,5 +1,7 @@ /** - * Hachette injecting policy to page using webRequest + * This file is part of Haketilo. + * + * Function: Injecting policy to page by modifying HTTP headers. * * Copyright (C) 2021 Wojtek Kosior * Copyright (C) 2021 jahoti @@ -8,186 +10,39 @@ /* * IMPORTS_START - * IMPORT TYPE_PREFIX - * IMPORT get_storage - * IMPORT browser - * IMPORT is_chrome - * IMPORT is_mozilla - * IMPORT gen_unique - * IMPORT gen_nonce - * IMPORT is_privileged_url - * IMPORT url_item - * IMPORT url_extract_target - * IMPORT sign_policy - * IMPORT query_best - * IMPORT sanitize_csp_header + * IMPORT make_csp_rule + * IMPORT csp_header_regex + * Re-enable the import below once nonce stuff here is ready + * !mport gen_nonce * IMPORTS_END */ -var storage; - -const csp_header_names = new Set([ - "content-security-policy", - "x-webkit-csp", - "x-content-security-policy" -]); - -/* TODO: variable no longer in use; remove if not needed */ -const unwanted_csp_directives = new Set([ - "report-to", - "report-uri", - "script-src", - "script-src-elem", - "prefetch-src" -]); - -const report_only = "content-security-policy-report-only"; - -function url_inject(details) -{ - if (is_privileged_url(details.url)) - return; - - const targets = url_extract_target(details.url); - if (targets.current) - return; - - /* Redirect; update policy */ - if (targets.policy) - targets.target = ""; - - let [pattern, settings] = query_best(storage, targets.base_url); - /* Defaults */ - if (!pattern) - settings = {}; - - const policy = encodeURIComponent( - JSON.stringify({ - allow: settings.allow, - nonce: gen_nonce(), - base_url: targets.base_url - }) - ); - - return { - redirectUrl: [ - targets.base_url, - '#', sign_policy(policy, new Date()), policy, - targets.target, - targets.target2 - ].join("") - }; -} - -function headers_inject(details) +function inject_csp_headers(headers, policy) { - const targets = url_extract_target(details.url); - /* Block mis-/unsigned requests */ - if (!targets.current) - return {cancel: true}; - - let orig_csp_headers = is_chrome ? null : []; - let headers = []; - let csp_headers = is_chrome ? headers : []; + let csp_headers; - const rule = `'nonce-${targets.policy.nonce}'`; - const block = !targets.policy.allow; + if (policy.payload) { + headers = headers.filter(h => !csp_header_regex.test(h.name)); - for (const header of details.responseHeaders) { - if (!csp_header_names.has(header)) { - /* Remove headers that only snitch on us */ - if (header.name.toLowerCase() === report_only && block) - continue; - headers.push(header); + // TODO: make CSP rules with nonces and facilitate passing them to + // content scripts via dynamic content script registration or + // synchronous XHRs - /* If these are the original CSP headers, use them instead */ - /* Test based on url_extract_target() in misc.js */ - if (is_mozilla && header.name === "x-orig-csp") { - let index = header.value.indexOf('%5B'); - if (index === -1) - continue; - - let sig = header.value.substring(0, index); - let data = header.value.substring(index); - if (sig !== sign_policy(data, 0)) - continue; - - /* Confirmed- it's the originals, smuggled in! */ - try { - data = JSON.parse(decodeURIComponent(data)); - } catch (e) { - /* This should not be reached - - it's our self-produced valid JSON. */ - console.log("Unexpected internal error - invalid JSON smuggled!", e); - } - - orig_csp_headers = csp_headers = null; - for (const header of data) - headers.push(sanitize_csp_header(header, rule, block)); - } - } else if (is_chrome || !orig_csp_headers) { - csp_headers.push(sanitize_csp_header(header, rule, block)); - if (is_mozilla) - orig_csp_headers.push(header); - } - } - - if (orig_csp_headers) { - /** Smuggle in the original CSP headers for future use. - * These are signed with a time of 0, as it's not clear there - * is a limit on how long Firefox might retain these headers in - * the cache. - */ - orig_csp_headers = encodeURIComponent(JSON.stringify(orig_csp_headers)); - headers.push({ - name: "x-orig-csp", - value: sign_policy(orig_csp_headers, 0) + orig_csp_headers - }); - - headers = headers.concat(csp_headers); + // policy.nonce = gen_nonce(); } - /* To ensure there is a CSP header if required */ - if (block) { + if (!policy.allow && (policy.nonce || !policy.payload)) { headers.push({ name: "content-security-policy", - value: `script-src ${rule}; script-src-elem ${rule}; ` + - "script-src-attr 'none'; prefetch-src 'none';" + value: make_csp_rule(policy) }); } - return {responseHeaders: headers}; -} - -async function start_policy_injector() -{ - storage = await get_storage(); - - let extra_opts = ["blocking", "responseHeaders"]; - if (is_chrome) - extra_opts.push("extraHeaders"); - - browser.webRequest.onBeforeRequest.addListener( - url_inject, - { - urls: ["<all_urls>"], - types: ["main_frame", "sub_frame"] - }, - ["blocking"] - ); - - browser.webRequest.onHeadersReceived.addListener( - headers_inject, - { - urls: ["<all_urls>"], - types: ["main_frame", "sub_frame"] - }, - extra_opts - ); + return headers; } /* * EXPORTS_START - * EXPORT start_policy_injector + * EXPORT inject_csp_headers * EXPORTS_END */ diff --git a/background/storage.js b/background/storage.js index c2160b0..a4e626a 100644 --- a/background/storage.js +++ b/background/storage.js @@ -1,5 +1,7 @@ /** - * Hachette storage manager + * This file is part of Haketilo. + * + * Function: Storage manager. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. @@ -7,7 +9,7 @@ /* * IMPORTS_START - * IMPORT TYPE_PREFIX + * IMPORT raw_storage * IMPORT TYPE_NAME * IMPORT list_prefixes * IMPORT make_lock @@ -15,76 +17,17 @@ * IMPORT unlock * IMPORT make_once * IMPORT browser - * IMPORT is_chrome * IMPORT observables * IMPORTS_END */ var exports = {}; -/* We're yet to decide how to handle errors... */ - -/* Here are some basic wrappers for storage API functions */ - -async function get(key) -{ - try { - /* Fix for fact that Chrome does not use promises here */ - let promise = is_chrome ? - new Promise((resolve, reject) => - chrome.storage.local.get(key, - val => resolve(val))) : - browser.storage.local.get(key); - - return (await promise)[key]; - } catch (e) { - console.log(e); - } -} - -async function set(key, value) -{ - try { - return browser.storage.local.set({[key]: value}); - } catch (e) { - console.log(e); - } -} - -async function setn(keys_and_values) -{ - let obj = Object(); - while (keys_and_values.length > 1) { - let value = keys_and_values.pop(); - let key = keys_and_values.pop(); - obj[key] = value; - } - - try { - return browser.storage.local.set(obj); - } catch (e) { - console.log(e); - } -} - -async function set_var(name, value) -{ - return set(TYPE_PREFIX.VAR + name, value); -} - -async function get_var(name) -{ - return get(TYPE_PREFIX.VAR + name); -} - -/* - * A special case of persisted variable is one that contains list - * of items. - */ +/* A special case of persisted variable is one that contains list of items. */ async function get_list_var(name) { - let list = await get_var(name); + let list = await raw_storage.get_var(name); return list === undefined ? [] : list; } @@ -97,7 +40,7 @@ async function list(prefix) let map = new Map(); for (let item of await get_list_var(name)) - map.set(item, await get(prefix + item)); + map.set(item, await raw_storage.get(prefix + item)); return {map, prefix, name, observable: observables.make(), lock: make_lock()}; @@ -175,19 +118,19 @@ async function set_item(item, value, list) } async function _set_item(item, value, list) { - let key = list.prefix + item; - let old_val = list.map.get(item); + const key = list.prefix + item; + const old_val = list.map.get(item); + const set_obj = {[key]: value}; if (old_val === undefined) { - let items = list_items(list); + const items = list_items(list); items.push(item); - await setn([key, value, "_" + list.name, items]); - } else { - await set(key, value); + set_obj["_" + list.name] = items; } - list.map.set(item, value) + await raw_storage.set(set_obj); + list.map.set(item, value); - let change = { + const change = { prefix : list.prefix, item, old_val, @@ -212,20 +155,21 @@ async function remove_item(item, list) } async function _remove_item(item, list) { - let old_val = list.map.get(item); + const old_val = list.map.get(item); if (old_val === undefined) return; - let key = list.prefix + item; - let items = list_items(list); - let index = items.indexOf(item); + const items = list_items(list); + const index = items.indexOf(item); items.splice(index, 1); - await setn([key, undefined, "_" + list.name, items]); - + await raw_storage.set({ + [list.prefix + item]: undefined, + ["_" + list.name]: items + }); list.map.delete(item); - let change = { + const change = { prefix : list.prefix, item, old_val, @@ -247,11 +191,11 @@ async function replace_item(old_item, new_item, list, new_val=undefined) } async function _replace_item(old_item, new_item, list, new_val=undefined) { - let old_val = list.map.get(old_item); + const old_val = list.map.get(old_item); if (new_val === undefined) { if (old_val === undefined) return; - new_val = old_val + new_val = old_val; } else if (new_val === old_val && new_item === old_item) { return old_val; } @@ -261,17 +205,18 @@ async function _replace_item(old_item, new_item, list, new_val=undefined) return old_val; } - let new_key = list.prefix + new_item; - let old_key = list.prefix + old_item; - let items = list_items(list); - let index = items.indexOf(old_item); + const items = list_items(list); + const index = items.indexOf(old_item); items[index] = new_item; - await setn([old_key, undefined, new_key, new_val, - "_" + list.name, items]); + await raw_storage.set({ + [list.prefix + old_item]: undefined, + [list.prefix + new_item]: new_val, + ["_" + list.name]: items + }); list.map.delete(old_item); - let change = { + const change = { prefix : list.prefix, item : old_item, old_val, diff --git a/background/storage_server.js b/background/storage_server.js index 2252eb5..73126d4 100644 --- a/background/storage_server.js +++ b/background/storage_server.js @@ -1,5 +1,7 @@ /** - * Hachette storage through connection (server side) + * This file is part of Haketilo. + * + * Function: Storage through messages (server side). * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. diff --git a/background/stream_filter.js b/background/stream_filter.js new file mode 100644 index 0000000..e5d124c --- /dev/null +++ b/background/stream_filter.js @@ -0,0 +1,214 @@ +/** + * This file is part of Haketilo. + * + * Function: Modifying a web page using the StreamFilter API. + * + * Copyright (C) 2018 Giorgio Maone <giorgio@maone.net> + * Copyright (C) 2021 Wojtek Kosior + * Redistribution terms are gathered in the `copyright' file. + * + * Derived from `bg/ResponseProcessor.js' and `bg/ResponseMetaData.js' + * in LibreJS. + */ + +/* + * IMPORTS_START + * IMPORT browser + * IMPORT csp_header_regex + * IMPORTS_END + */ + +function validate_encoding(charset) +{ + try { + new TextDecoder(); + return charset; + } catch(e) { + return undefined; + } +} + +function is_content_type_header(header) +{ + header.name.toLowerCase().trim() === "content-type"; +} + +const charset_reg = /;\s*charset\s*=\s*([\w-]+)/i; + +function properties_from_headers(headers) +{ + const properties = {}; + + for (const header of headers.filter(is_content_type_header)) { + const match = charset_reg.exec(header.value); + if (!properties.detected_charset && validate_encoding(match[1])) + properties.detected_charset = match[1]; + + if (/html/i.test(header.value)) + properties.html = true; + } + + return properties; +} + +const UTF8_BOM = [0xef, 0xbb, 0xbf]; +const BOMs = [ + [UTF8_BOM, "utf-8"], + [[0xfe, 0xff], "utf-16be"], + [[0xff, 0xfe], "utf-16le"] +]; + +function charset_from_BOM(data) +{ + for (const [BOM, charset] of BOMs) { + if (BOM.reduce((ac, byte, i) => ac && byte === data[i], true)) + return charset; + } + + return ""; +} + +const charset_attrs = + ['charset', 'http-equiv="content-type"', 'content*="charset"']; +const charset_meta_selector = + charset_attrs.map(a => `head>meta[${a}]`).join(", "); + +function charset_from_meta_tags(doc) +{ + for (const meta of doc.querySelectorAll(charset_meta_selector)) { + const maybe_charset = meta.getAttribute("charset"); + if (maybe_charset && validate_encoding(maybe_charset)) + return maybe_charset; + + const match = charset_reg.exec(meta.getAttribute("content")); + if (match && validate_encoding(match[1])) + return match[1]; + } + + return undefined; +} + +function create_decoder(properties, data) +{ + let charset = charset_from_BOM(data) || properties.detected_charset; + if (!charset && data.indexOf(0) !== -1) { + console.debug("Warning: zeroes in bytestream, probable cached encoding mismatch. Trying to decode it as UTF-16.", + properties); + return new TextDecoder("utf-16be"); + } + + /* Missing HTTP charset, sniffing in content... */ + /* + * TODO: I recall there is some standard saying how early in the doc the + * charset has to be specified. We could process just this part of data. + */ + const text = new TextDecoder("latin1").decode(data, {stream: true}); + properties.html = properties.html || /html/i.test(text); + + if (properties.html) { + const tmp_doc = new DOMParser().parseFromString(text, "text/html"); + charset = charset_from_meta_tags(tmp_doc); + } + + return new TextDecoder(charset || "latin1"); +} + +function may_define_csp_rules(html) +{ + const doc = new DOMParser().parseFromString(html, "text/html"); + + for (const meta of doc.querySelectorAll("head>meta[http-equiv]")) { + if (csp_header_regex.test(meta.httpEquiv) && meta.content) + return true; + } + + /* + * Even if no naughty `<meta>' tags were found, subsequent chunk of HTML + * data could add some. Before we return `false' we need to be sure we + * reached the start of `<body>' where `<meta>' tags are no longer valid. + */ + + if (doc.documentElement.nextSibling || doc.body.nextSibling || + doc.body.childNodes.length > 1) + return false; + + if (!doc.body.firstChild) + return true; + + if (doc.body.firstChild.nodeName !== "#text") + return false; + + return /^(<\/|&#|.)$/.test(doc.body.firstChild.wholeText); +} + +function filter_data(properties, event) +{ + const data = new Uint8Array(event.data); + let first_chunk = false; + if (!properties.decoder) { + first_chunk = true; + properties.decoder = create_decoder(properties, data); + properties.encoder = new TextEncoder(); + } + + let decoded = properties.decoder.decode(data); + + /* Force UTF-8, this is the only encoding we can produce. */ + if (first_chunk) + properties.filter.write(new Uint8Array(UTF8_BOM)); + + if (first_chunk && may_define_csp_rules(decoded)) { + /* + * HAX! Our content scripts that execute at `document_start' will always + * run before the first script in the document, but under Mozilla some + * `<meta>' tags might already be loaded at that point. Here we inject a + * dummy `<script>' at the beginning (before any `<meta>' tags) that + * will force `document_start' to happen earlier. This way our content + * scripts will be able to sanitize `http-equiv' tags with CSP rules + * that would otherwise stop our injected scripts from executing. + * + * As we want to only process HTML files that happen to have naughty + * `<meta>' tags in `<head>', we use a DOMParser-based heuristic in + * `may_define_rules()'. We don't do any additional MIME sniffing as it + * is too unreliable (and our heuristic will likely mark non-HTML files + * as harmless anyway). + */ + + const dummy_script = `<script>null</script>`; + const doctype_decl = /^(\s*<!doctype[^<>"']*>)?/i.exec(decoded)[0]; + decoded = doctype_decl + dummy_script + + decoded.substring(doctype_decl.length); + } + + properties.filter.write(properties.encoder.encode(decoded)); + + if (properties.decoder.encoding === "utf-8") + properties.filter.disconnect(); +} + +function apply_stream_filter(details, headers, policy) +{ + if (!policy.payload) + return headers; + + const properties = properties_from_headers(headers); + + properties.filter = + browser.webRequest.filterResponseData(details.requestId); + + properties.filter.ondata = event => filter_data(properties, event); + properties.filter.onstop = () => properties.filter.close(); + + /* + * In the future we might consider modifying the headers that specify + * encoding. For now we are not yet doing it, though. However, we + * prepend the data with UTF-8 BOM which should be enough. + */ + return headers; +} + +/* + * EXPORTS_START + * EXPORT apply_stream_filter + * EXPORTS_END + */ @@ -3,279 +3,172 @@ # Copyright (C) 2021 Wojtek Kosior # Redistribution terms are gathered in the `copyright' file. -. ./shell_utils.sh - -handle_export_line() { - if [ "x$1" = "xEXPORTS_START" ]; then - if [ "$STATE" = "before_block" ]; then - STATE="in_block" - fi - elif [ "x$1" = "xEXPORT" ]; then - if [ "$STATE" != "in_block" ]; then - return - fi - - EXPORTCODE="${EXPORTCODE}window.killtheweb.$2 = $2;$ENDL" - - PREVIOUS_FILE="$(map_get EXPORTS $2)" - if [ "x$PREVIOUS_FILE" != "x" ]; then - errcho "export $2 present in both $PREVIOUS_FILE and $FILE" - return 1 - fi - - map_set_instr EXPORTS $2 "$FILE" - - elif [ "x$1" = "xEXPORTS_END" ]; then - if [ "$STATE" = "in_block" ]; then - STATE="after_block" - fi - fi -} - -translate_exports() { - STATE="before_block" - EXPORTCODE='' - - while read EXPORT_LINE; do - handle_export_line $EXPORT_LINE || return 1 - done - - map_set_instr EXPORTCODES $FILEKEY "$EXPORTCODE" -} - -add_exports() { - FILE="$1" - FILEKEY="$(sanitize "$FILE")" - - eval "$(grep -o 'EXPORT.\+' "$1" | translate_exports || exit 1)" -} - -handle_import_line() { - if [ "x$1" = "xIMPORTS_START" ]; then - if [ "$STATE" = "before_block" ]; then - STATE="in_block" - fi - elif [ "x$1" = "xIMPORT" ]; then - if [ "$STATE" != "in_block" ]; then - return - fi - - IMPORTCODE="${IMPORTCODE}const $2 = window.killtheweb.$2;$ENDL" - - IMPORTS="$IMPORTS $2" - - elif [ "x$1" = "xIMPORTS_END" ]; then - if [ "$STATE" = "in_block" ]; then - STATE="after_block" - fi - fi -} - -translate_imports() { - STATE="before_block" - IMPORTCODE='' - IMPORTS='' - - while read IMPORT_LINE; do - handle_import_line $IMPORT_LINE || return 1 - done - - map_set_instr IMPORTCODES $FILEKEY "$IMPORTCODE" - map_set_instr IMPORTS $FILEKEY "$IMPORTS" -} - -add_imports() { - FILE="$1" - FILEKEY="$(sanitize "$FILE")" - - eval "$(grep -o 'IMPORT.\+' "$1" | translate_imports || exit 1)" -} - -compute_scripts_list_rec() { - local FILE="$1" - local FILEKEY=$(sanitize "$1") - - local FILESTATE="$(map_get FILESTATES $FILEKEY)" - if [ "xprocessed" = "x$FILESTATE" ]; then - return - fi - if [ "xprocessing" = "x$FILESTATE" ]; then - errcho "import loop on $FILE" - return 1 - fi - - USED="$USED $FILEKEY" - - map_set FILESTATES $FILEKEY "processing" +set -e - local IMPORT - for IMPORT in $(map_get IMPORTS $FILEKEY); do - NEXT_FILE="$(map_get EXPORTS $IMPORT)" - if [ "x" = "x$NEXT_FILE" ]; then - errcho "nothing exports $IMPORT, required by $FILE" - return 1 - fi - if ! compute_scripts_list_rec "$NEXT_FILE"; then - errcho "when satisfying $IMPORT for $FILE" - return 1 - fi - done - - [ "x$FILE" = "xexports_init.js" ] || echo $FILE # exports_init.js is hardcoded to load first; the entire export system depends on it - map_set FILESTATES $FILEKEY "processed" -} - -compute_scripts_list() { - USED='' - echo COMPUTED_SCRIPTS=\"exports_init.js - compute_scripts_list_rec "$1" - echo \" - - for FILEKEY in $USED; do - map_set_instr USED $FILEKEY yes - done -} +. ./shell_utils.sh as_json_list() { while true; do if [ "x" = "x$2" ]; then - echo -n '\\n'"\t\t\"$1\""'\\n\t' + printf '\\n\t\t"%s"\\n\t' "$1" return fi - echo -n '\\n'"\t\t\"$1\"," + printf '\\n\t\t"%s",' "$1" shift done } as_html_list() { while [ "x" != "x$1" ]; do - echo -n '\\n'" <script src=\"/$1\"></script>" + printf '\\n <script src="/%s"></script>' "$1" shift done } -set_browser() { - if [ "x$1" = "xmozilla" -o "x$1" = "xchromium" ]; then - BROWSER="$1" - else - errcho "usage: $0 mozilla|chromium" - exit 1 - fi +compute_scripts() { + local DIRS="$1" + local ROOT_SCRIPT="$2" + + local AVAILABLE="$(find $DIRS -name '[^.#]*.js')" + + awk -f compute_scripts.awk script_dependencies "$ROOT_SCRIPT" $AVAILABLE } -main() { - set_browser "$1" +build_main() { + local ALL_SCRIPTDIRS='background html common content' + + local ALL_SCRIPTS_AVAILABLE="$(find $ALL_SCRIPTDIRS -name '[^.#]*.js')" + + local SCRIPT + for SCRIPT in $ALL_SCRIPTS_AVAILABLE; do + map_set SCRIPTS_UNUSED $(sanitize $SCRIPT) yes + done + + local ROOT=background/main.js + local SCRIPTS_BG="$( compute_scripts 'common/ background/' $ROOT)" + + local ROOT=content/main.js + local SCRIPTS_CONTENT="$( compute_scripts 'common/ content/' $ROOT)" - # placate importers of these, as they are exported by the yet-to-be-created exports_init.js - EXPORTS__browser=exports_init.js - EXPORTS__is_chrome=exports_init.js - EXPORTS__is_mozilla=exports_init.js + local ROOT=html/display-panel.js + local SCRIPTS_POPUP="$( compute_scripts 'common/ html/' $ROOT)" - SCRIPTDIRS='background html common content' + local ROOT=html/options_main.js + local SCRIPTS_OPTIONS="$( compute_scripts 'common/ html/' $ROOT)" - SCRIPTS=$(find $SCRIPTDIRS -name '[^.#]*.js') + local BGSCRIPTS="$( as_json_list $SCRIPTS_BG )" + local CONTENTSCRIPTS="$( as_json_list $SCRIPTS_CONTENT )" + local POPUPSCRIPTS="$( as_html_list $SCRIPTS_POPUP )" + local OPTIONSSCRIPTS="$( as_html_list $SCRIPTS_OPTIONS )" - for SCRIPT in $SCRIPTS; do - add_exports $SCRIPT - add_imports $SCRIPT + for SCRIPT in $SCRIPTS_BG $SCRIPTS_CONTENT $SCRIPTS_POPUP $SCRIPTS_OPTIONS + do + map_del SCRIPTS_UNUSED $(sanitize $SCRIPT) done - eval "$(compute_scripts_list background/main.js || exit 1)" - BGSCRIPTS="$(as_json_list $COMPUTED_SCRIPTS)" - eval "$(compute_scripts_list content/main.js || exit 1)" - CONTENTSCRIPTS="$(as_json_list $COMPUTED_SCRIPTS)" - eval "$(compute_scripts_list html/display-panel.js || exit 1)" - POPUPSCRIPTS="$(as_html_list $COMPUTED_SCRIPTS)" - eval "$(compute_scripts_list html/options_main.js || exit 1)" - OPTIONSSCRIPTS="$(as_html_list $COMPUTED_SCRIPTS)" - - BUILDDIR=build_$BROWSER - rm -rf $BUILDDIR - mkdir $BUILDDIR - for DIR in $(find $SCRIPTDIRS -type d); do - mkdir -p $BUILDDIR/$DIR + for DIR in $(find $ALL_SCRIPTDIRS -type d); do + mkdir -p "$BUILDDIR"/$DIR done - CHROMIUM_KEY='' + CHROMIUM_UPDATE_URL='' GECKO_APPLICATIONS='' + if [ "x$UPDATE_URL" != x ]; then + UPDATE_URL=",\n \"update_url\": \"$UPDATE_URL\"" + fi + if [ "$BROWSER" = "chromium" ]; then - CHROMIUM_KEY="\n\ -\n\ - // WARNING!!!\n\ - // EACH USER SHOULD REPLACE \"key\" WITH A UNIQUE VALUE!!!\n\ - // OTHERWISE, SECURITY CAN BE TRIVIALLY COMPROMISED!\n\ - //\n\ - // A unique key can be generated with:\n\ - // $ ssh-keygen -f /path/to/new/key.pem -t rsa -b 1024\n\ - //\n\ - // Only relevant to users of chrome-based browsers.\n\ - // Users of Firefox forks are safe.\n\ -\n\ - \"key\": \"b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAlwAAAAdzc2gtcnNhAAAAAwEAAQAAAIEA+0GT5WNmRRo8e5tL9+BmNtY6aBPwLIgbPnLShYBMSR40iYwLTsccrkwBXb3bs1o4p6q5WJugI8Lsia+GXZc/XHGFkq7D1aWiTxlJLs8z0JC2TQ2/yatYmBMchogYGeeUfP7aI7JJZwpATts+VhIvgga/4FYj+DijMIEpwdckqFEAAAII4Dh7HOA4exwAAAAHc3NoLXJzYQAAAIEA+0GT5WNmRRo8e5tL9+BmNtY6aBPwLIgbPnLShYBMSR40iYwLTsccrkwBXb3bs1o4p6q5WJugI8Lsia+GXZc/XHGFkq7D1aWiTxlJLs8z0JC2TQ2/yatYmBMchogYGeeUfP7aI7JJZwpATts+VhIvgga/4FYj+DijMIEpwdckqFEAAAADAQABAAAAgEHB5/MhEKMFOs8e1cMJ97ZiWubiUPlWpcqyQmauLUj1nspg3JTBh8AWJEVkaxuFgU5gYCHQmRjC6yUdywyziOEkFA4r/WpX4WmbIe+GQHRHhitLN0dgF8N6/fVNOoa5StTdfZqyl23pVXyepoDNjrJFKyupqPMmpwfH5lGr9RwBAAAAQG76HflB/5j8P2YgIYX6dQT4Ei0SqiIjNVy7jFJUQDKSJg/PYkedE02JZJBJPcMYxEJUxXtMgq+upamNILfkmY0AAABBAP4v0O5dqjy16xDDFzb4DPNAcw5Za9KJaXKVkUuKXMNZOKTR0RC/upjNTmttY980RKdIx5zA25dO8cx563bSDIsAAABBAP0MaOpBiai/eRmLqhlthHODa+Mur6W3uc9PyhWhgDBjLNMR/doaYeyfVKxtIiN3a+HkN++G+vbokRweQv++bhMAAAANdXJ6QGxvY2FsaG9zdAECAwQFBg==\"," + CHROMIUM_UPDATE_URL="$UPDATE_URL" else GECKO_APPLICATIONS="\n\ \"applications\": {\n\ \"gecko\": {\n\ \"id\": \"{6fe13369-88e9-440f-b837-5012fb3bedec}\",\n\ - \"strict_min_version\": \"60.0\"\n\ + \"strict_min_version\": \"60.0\"$UPDATE_URL\n\ }\n\ }," fi sed "\ s^_GECKO_APPLICATIONS_^$GECKO_APPLICATIONS^ -s^_CHROMIUM_KEY_^$CHROMIUM_KEY^ +s^_CHROMIUM_UPDATE_URL_^$CHROMIUM_UPDATE_URL^ s^_BGSCRIPTS_^$BGSCRIPTS^ s^_CONTENTSCRIPTS_^$CONTENTSCRIPTS^" \ - < manifest.json > $BUILDDIR/manifest.json + < manifest.json > "$BUILDDIR"/manifest.json ./process_html_file.sh html/display-panel.html | sed "s^_POPUPSCRIPTS_^$POPUPSCRIPTS^" \ - > $BUILDDIR/html/display-panel.html + > "$BUILDDIR"/html/display-panel.html ./process_html_file.sh html/options.html | sed "s^_OPTIONSSCRIPTS_^$OPTIONSSCRIPTS^" \ - > $BUILDDIR/html/options.html + > "$BUILDDIR"/html/options.html - for FILE in $SCRIPTS; do + for FILE in $ALL_SCRIPTS_AVAILABLE; do FILEKEY=$(sanitize "$FILE") - if [ "xyes" != "x$(map_get USED $FILEKEY)" ]; then - errcho "WARNING! $FILE not used" + if [ "x$(map_get SCRIPTS_UNUSED $FILEKEY)" = "xyes" ]; then + printf 'WARNING! %s not used\n' "$FILE" >&2 else - (echo "\ -\"use strict\"; + awk -f compute_scripts.awk wrapped_code "$FILE" > "$BUILDDIR"/$FILE + fi + done -(() => { -$(map_get IMPORTCODES $FILEKEY) + # A hack to insert the contents of default_settings.json at the appropriate + # location in background/main.js. Uses an internal sed expression to escape + # and indent the JSON file for use in the external sed expression. + sed -i 's/^ `DEFAULT SETTINGS`$/'"$(sed -E 's/([\\\&\/])/\\\1/g; s/^/ /; s/$/\\/' < default_settings.json) "/g "$BUILDDIR"/background/main.js -"; + if [ "$BROWSER" = "chromium" ]; then + cp CHROMIUM_exports_init.js "$BUILDDIR"/exports_init.js + else + cp MOZILLA_exports_init.js "$BUILDDIR"/exports_init.js + fi + + cp -r copyright licenses/ "$BUILDDIR" + cp dummy "$BUILDDIR" + cp html/*.css "$BUILDDIR"/html + mkdir "$BUILDDIR"/icons + cp icons/*.png "$BUILDDIR"/icons -# A hack to insert the contents of default_settings.json at the appropriate location in background/main.js -if [ "$FILE" = "background/main.js" ]; then - # Uses an internal sed expression to escape and indent the JSON file for use in the external sed expression - sed 's/^ `DEFAULT SETTINGS`$/'"$(sed -E 's/([\\\&\/])/\\\1/g; s/^/ /; s/$/\\/' < default_settings.json) "/g < "$FILE" -else - cat $FILE -fi + if [ "$BROWSER" = "chromium" ]; then + for MOZILLA_FILE in $(find "$BUILDDIR" -name "MOZILLA_*"); do + printf '\n' > "$MOZILLA_FILE" + done + fi + if [ "$BROWSER" = "mozilla" ]; then + for CHROMIUM_FILE in $(find "$BUILDDIR" -name "CHROMIUM_*"); do + printf '\n' > "$CHROMIUM_FILE" + done + fi +} -echo " +print_usage() { + printf 'usage: %s mozilla|chromium [source directory] [update url]\n' \ + "$0" >&2 +} -$(map_get EXPORTCODES $FILEKEY) -})();") > $BUILDDIR/$FILE - fi - done +main() { + if [ "x$1" = "xmozilla" -o "x$1" = "xchromium" ]; then + BROWSER=$1 + else + print_usage + exit 1 + fi - if [ "$BROWSER" = "chromium" ]; then - echo "window.killtheweb={is_chrome: true, browser: window.chrome};" > $BUILDDIR/exports_init.js + SRCDIR="${2:-.}" + if [ -d "$SRCDIR" ]; then + BUILDDIR="$(realpath $BROWSER-unpacked)" + rm -rf "$BUILDDIR" + mkdir "$BUILDDIR" + cd "$SRCDIR" else - echo "window.killtheweb={is_mozilla: true, browser: this.browser};" > $BUILDDIR/exports_init.js + print_usage + exit 2 fi - cp -r copyright licenses/ $BUILDDIR - cp html/*.css $BUILDDIR/html - mkdir $BUILDDIR/icons - cp icons/*.png $BUILDDIR/icons + UPDATE_URL="$3" + + build_main } main "$@" diff --git a/common/ajax.js b/common/ajax.js index 8082bbe..7269a8a 100644 --- a/common/ajax.js +++ b/common/ajax.js @@ -1,6 +1,7 @@ /** - * part of Hachette - * Wrapping XMLHttpRequest into a Promise. + * This file is part of Haketilo. + * + * Function: Wrapping XMLHttpRequest into a Promise. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. diff --git a/common/connection_types.js b/common/connection_types.js index 88c6964..3e9df56 100644 --- a/common/connection_types.js +++ b/common/connection_types.js @@ -1,5 +1,7 @@ /** - * Hachette background scripts message connection types "enum" + * This file is part of Haketilo. + * + * Function: Define an "enum" of message connection types. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. diff --git a/common/lock.js b/common/lock.js index 822ad1b..6cf0835 100644 --- a/common/lock.js +++ b/common/lock.js @@ -1,5 +1,7 @@ /** - * Hachette lock (aka binary semaphore aka mutex) + * This file is part of Haketilo. + * + * Function: Implement a lock (aka binary semaphore aka mutex). * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. diff --git a/common/message_server.js b/common/message_server.js index ea40487..c8c6696 100644 --- a/common/message_server.js +++ b/common/message_server.js @@ -1,5 +1,7 @@ /** - * Hachette message server + * This file is part of Haketilo. + * + * Function: Message server. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. diff --git a/common/misc.js b/common/misc.js index 3c7dc46..5b0addb 100644 --- a/common/misc.js +++ b/common/misc.js @@ -1,5 +1,7 @@ /** - * Hachette miscellaneous operations refactored to a separate file + * This file is part of Haketilo. + * + * Function: Miscellaneous operations refactored to a separate file. * * Copyright (C) 2021 Wojtek Kosior * Copyright (C) 2021 jahoti @@ -8,9 +10,7 @@ /* * IMPORTS_START - * IMPORT sha256 * IMPORT browser - * IMPORT is_chrome * IMPORT TYPE_NAME * IMPORT TYPE_PREFIX * IMPORTS_END @@ -38,94 +38,27 @@ function Uint8toHex(data) return returnValue; } -function gen_nonce(length) // Default 16 +function gen_nonce(length=16) { - let randomData = new Uint8Array(length || 16); + let randomData = new Uint8Array(length); crypto.getRandomValues(randomData); return Uint8toHex(randomData); } -function gen_unique(url) -{ - return sha256(get_secure_salt() + url); -} - -function get_secure_salt() -{ - if (is_chrome) - return browser.runtime.getManifest().key.substring(0, 50); - else - return browser.runtime.getURL("dummy"); -} - -/* - * stripping url from query and target (everything after `#' or `?' - * gets removed) - */ -function url_item(url) -{ - let url_re = /^([^?#]*).*$/; - let match = url_re.exec(url); - return match[1]; -} - -/* - * Assume a url like: - * https://example.com/green?illuminati=confirmed#<injected-policy>#winky - * This function will make it into an object like: - * { - * "base_url": "https://example.com/green?illuminati=confirmed", - * "target": "#<injected-policy>", - * "target2": "#winky", - * "policy": <injected-policy-as-js-object>, - * "current": <boolean-indicating-whether-policy-url-matches> - * } - * In case url doesn't have 2 #'s, target2 and target can be set to undefined. - */ -function url_extract_target(url) -{ - const url_re = /^([^#]*)((#[^#]*)(#.*)?)?$/; - const match = url_re.exec(url); - const targets = { - base_url: match[1], - target: match[3] || "", - target2: match[4] || "" - }; - if (!targets.target) - return targets; - - /* %7B -> { */ - const index = targets.target.indexOf('%7B'); - if (index === -1) - return targets; - - const now = new Date(); - const sig = targets.target.substring(1, index); - const policy = targets.target.substring(index); - if (sig !== sign_policy(policy, now) && - sig !== sign_policy(policy, now, -1)) - return targets; - - try { - targets.policy = JSON.parse(decodeURIComponent(policy)); - targets.current = targets.policy.base_url === targets.base_url; - } catch (e) { - /* This should not be reached - it's our self-produced valid JSON. */ - console.log("Unexpected internal error - invalid JSON smuggled!", e); - } - - return targets; -} - -/* csp rule that blocks all scripts except for those injected by us */ -function csp_rule(nonce) +/* CSP rule that blocks scripts according to policy's needs. */ +function make_csp_rule(policy) { - let rule = `script-src 'nonce-${nonce}';`; - if (is_chrome) - rule += `script-src-elem 'nonce-${nonce}';`; + let rule = "prefetch-src 'none'; script-src-attr 'none';"; + const script_src = policy.nonce !== undefined ? + `'nonce-${policy.nonce}'` : "'none'"; + rule += ` script-src ${script_src}; script-src-elem ${script_src};`; return rule; } +/* Check if some HTTP header might define CSP rules. */ +const csp_header_regex = + /^\s*(content-security-policy|x-webkit-csp|x-content-security-policy)/i; + /* * Print item together with type, e.g. * nice_name("s", "hello") → "hello (script)" @@ -143,17 +76,13 @@ function open_in_settings(prefix, name) window.open(url, "_blank"); } -/* Check if url corresponds to a browser's special page */ -function is_privileged_url(url) -{ - return !!/^(chrome(-extension)?|moz-extension):\/\/|^about:/i.exec(url); -} - -/* Sign a given policy for a given time */ -function sign_policy(policy, now, hours_offset) { - let time = Math.floor(now / 3600000) + (hours_offset || 0); - return gen_unique(time + policy); -} +/* + * Check if url corresponds to a browser's special page (or a directory index in + * case of `file://' protocol). + */ +const privileged_reg = + /^(chrome(-extension)?|moz-extension):\/\/|^about:|^file:\/\/.*\/$/; +const is_privileged_url = url => privileged_reg.test(url); /* Parse a CSP header */ function parse_csp(csp) { @@ -174,41 +103,7 @@ function parse_csp(csp) { return directives; } -/* Make CSP headers do our bidding, not interfere */ -function sanitize_csp_header(header, rule, block) -{ - const csp = parse_csp(header.value); - - if (block) { - /* No snitching */ - delete csp['report-to']; - delete csp['report-uri']; - - delete csp['script-src']; - delete csp['script-src-elem']; - - csp['script-src-attr'] = ["'none'"]; - csp['prefetch-src'] = ["'none'"]; - } - - if ('script-src' in csp) - csp['script-src'].push(rule); - else - csp['script-src'] = [rule]; - - if ('script-src-elem' in csp) - csp['script-src-elem'].push(rule); - else - csp['script-src-elem'] = [rule]; - - const new_policy = Object.entries(csp).map( - i => `${i[0]} ${i[1].join(' ')};` - ); - - return {name: header.name, value: new_policy.join('')}; -} - -/* Regexes and objest to use as/in schemas for parse_json_with_schema(). */ +/* Regexes and objects to use as/in schemas for parse_json_with_schema(). */ const nonempty_string_matcher = /.+/; const matchers = { @@ -223,15 +118,11 @@ const matchers = { /* * EXPORTS_START * EXPORT gen_nonce - * EXPORT gen_unique - * EXPORT url_item - * EXPORT url_extract_target - * EXPORT sign_policy - * EXPORT csp_rule + * EXPORT make_csp_rule + * EXPORT csp_header_regex * EXPORT nice_name * EXPORT open_in_settings * EXPORT is_privileged_url - * EXPORT sanitize_csp_header * EXPORT matchers * EXPORTS_END */ diff --git a/common/observable.js b/common/observable.js index 1fb0b0a..ab3b444 100644 --- a/common/observable.js +++ b/common/observable.js @@ -1,33 +1,28 @@ /** - * part of Hachette - * Facilitate listening to events + * This file is part of Haketilo. + * + * Function: Facilitate listening to (internal, self-generated) events. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. */ -function make() -{ - return new Set(); -} - -function subscribe(observable, cb) -{ - observable.add(cb); -} +const make = (value=undefined) => ({value, listeners: new Set()}); +const subscribe = (observable, cb) => observable.listeners.add(cb); +const unsubscribe = (observable, cb) => observable.listeners.delete(cb); -function unsubscribe(observable, cb) -{ - observable.delete(cb); -} +const silent_set = (observable, value) => observable.value = value; +const broadcast = (observable, ...values) => + observable.listeners.forEach(cb => cb(...values)); -function broadcast(observable, event) +function set(observable, value) { - for (const callback of observable) - callback(event); + const old_value = observable.value; + silent_set(observable, value); + broadcast(observable, value, old_value); } -const observables = {make, subscribe, unsubscribe, broadcast}; +const observables = {make, subscribe, unsubscribe, broadcast, silent_set, set}; /* * EXPORTS_START diff --git a/common/once.js b/common/once.js index 098b43f..93e842f 100644 --- a/common/once.js +++ b/common/once.js @@ -1,5 +1,8 @@ /** - * Hachette feature initialization promise + * This file is part of Haketilo. + * + * Function: Wrap APIs that depend on some asynchronous initialization into + * promises. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. diff --git a/common/patterns.js b/common/patterns.js index be7c650..635b128 100644 --- a/common/patterns.js +++ b/common/patterns.js @@ -1,187 +1,151 @@ /** - * Hachette operations on page url patterns + * This file is part of Haketilo. + * + * Function: Operations on page URL patterns. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. */ -const proto_re = "[a-zA-Z]*:\/\/"; -const domain_re = "[^/?#]+"; -const segments_re = "/[^?#]*"; -const query_re = "\\?[^#]*"; - -const url_regex = new RegExp(`\ -^\ -(${proto_re})\ -(${domain_re})\ -(${segments_re})?\ -(${query_re})?\ -#?.*\$\ -`); - -function deconstruct_url(url) -{ - const regex_match = url_regex.exec(url); - if (regex_match === null) - return undefined; +const MAX = { + URL_PATH_LEN: 12, + URL_PATH_CHARS: 255, + DOMAIN_LEN: 7, + DOMAIN_CHARS: 100 +}; - let [_, proto, domain, path, query] = regex_match; +const proto_regex = /^(\w+):\/\/(.*)$/; - domain = domain.split("."); - let path_trailing_dash = - path && path[path.length - 1] === "/"; - path = (path || "").split("/").filter(s => s !== ""); - path.unshift(""); +const user_re = "[^/?#@]+@" +const domain_re = "[.a-zA-Z0-9-]+"; +const path_re = "[^?#]*"; +const query_re = "\\??[^#]*"; - return {proto, domain, path, query, path_trailing_dash}; -} +const http_regex = new RegExp(`^(${domain_re})(${path_re})(${query_re}).*`); + +const file_regex = new RegExp(`^(${path_re}).*`); -/* Be sane: both arguments should be arrays of length >= 2 */ -function domain_matches(url_domain, pattern_domain) +const ftp_regex = new RegExp(`^(${user_re})?(${domain_re})(${path_re}).*`); + +function deconstruct_url(url, use_limits=true) { - const length_difference = url_domain.length - pattern_domain.length; - - for (let i = 1; i <= url_domain.length; i++) { - const url_part = url_domain[url_domain.length - i]; - const pattern_part = pattern_domain[pattern_domain.length - i]; - - if (pattern_domain.length === i) { - if (pattern_part === "*") - return length_difference === 0; - if (pattern_part === "**") - return length_difference > 0; - if (pattern_part === "***") - return true; - return length_difference === 0 && pattern_part === url_part; - } + const max = MAX; + if (!use_limits) { + for (key in MAX) + max[key] = Infinity; + } - if (pattern_part !== url_part) - return false; + const proto_match = proto_regex.exec(url); + if (proto_match === null) + throw `bad url '${url}'`; + + const deco = {proto: proto_match[1]}; + + if (deco.proto === "file") { + deco.path = file_regex.exec(proto_match[2])[1]; + } else if (deco.proto === "ftp") { + [deco.domain, deco.path] = ftp_regex.exec(proto_match[2]).slice(2, 4); + } else if (deco.proto === "http" || deco.proto === "https") { + const http_match = http_regex.exec(proto_match[2]); + if (!http_match) + return undefined; + [deco.domain, deco.path, deco.query] = http_match.slice(1, 4); + deco.domain = deco.domain.toLowerCase(); + } else { + throw `unsupported protocol in url '${url}'`; } - return pattern_domain.length === url_domain.length + 1 && - pattern_domain[0] === "***"; -} + deco.trailing_dash = deco.path[deco.path.length - 1] === "/"; -function path_matches(url_path, url_trailing_dash, - pattern_path, pattern_trailing_dash) -{ - const dashes_ok = !(pattern_trailing_dash && !url_trailing_dash); - - if (pattern_path.length === 0) - return url_path.length === 0 && dashes_ok; - - const length_difference = url_path.length - pattern_path.length; - - for (let i = 0; i < url_path.length; i++) { - if (pattern_path.length === i + 1) { - if (pattern_path[i] === "*") - return length_difference === 0; - if (pattern_path[i] === "**") { - return length_difference > 0 || - (url_path[i] === "**" && dashes_ok); - } - if (pattern_path[i] === "***") - return length_difference >= 0; - return length_difference === 0 && - pattern_path[i] === url_path[i] && dashes_ok; + if (deco.domain) { + if (deco.domain.length > max.DOMAIN_CHARS) { + const idx = deco.domain.indexOf(".", deco.domain.length - + max.DOMAIN_CHARS); + if (idx === -1) + deco.domain = []; + else + deco.domain = deco.domain.substring(idx + 1); + + deco.domain_truncated = true; } - if (pattern_path[i] !== url_path[i]) - return false; + if (deco.path.length > max.URL_PATH_CHARS) { + deco.path = deco.path.substring(0, deco.path.lastIndexOf("/")); + deco.path_truncated = true; + } } - return false; -} - -function url_matches(url, pattern) -{ - const url_deco = deconstruct_url(url); - const pattern_deco = deconstruct_url(pattern); - - if (url_deco === undefined || pattern_deco === undefined) { - console.log(`bad comparison: ${url} and ${pattern}`); - return false + if (typeof deco.domain === "string") { + deco.domain = deco.domain.split("."); + if (deco.domain.splice(0, deco.domain.length - max.DOMAIN_LEN).length + > 0) + deco.domain_truncated = true; } - if (pattern_deco.proto !== url_deco.proto) - return false; + deco.path = deco.path.split("/").filter(s => s !== ""); + if (deco.domain && deco.path.splice(max.URL_PATH_LEN).length > 0) + deco.path_truncated = true; - return domain_matches(url_deco.domain, pattern_deco.domain) && - path_matches(url_deco.path, url_deco.path_trailing_dash, - pattern_deco.path, pattern_deco.path_trailing_dash); + return deco; } -/* - * Call callback for every possible pattern that matches url. Return when there - * are no more patterns or callback returns false. - */ -function for_each_possible_pattern(url, callback) +function* each_domain_pattern(deco) { - const deco = deconstruct_url(url); - - if (deco === undefined) { - console.log("bad url format", url); - return; + for (let slice = 0; slice < deco.domain.length - 1; slice++) { + const domain_part = deco.domain.slice(slice).join("."); + const domain_wildcards = []; + if (slice === 0 && !deco.domain_truncated) + yield domain_part; + if (slice === 1 && !deco.domain_truncated) + yield "*." + domain_part; + if (slice > 1) + yield "**." + domain_part; + yield "***." + domain_part; } +} - for (let d_slice = 0; d_slice < deco.domain.length; d_slice++) { - const domain_part = deco.domain.slice(d_slice).join("."); - const domain_wildcards = []; - if (d_slice === 0) - domain_wildcards.push(""); - if (d_slice === 1) - domain_wildcards.push("*."); - if (d_slice > 0) - domain_wildcards.push("**."); - domain_wildcards.push("***."); - - for (const domain_wildcard of domain_wildcards) { - const domain_pattern = domain_wildcard + domain_part; - - for (let s_slice = deco.path.length; s_slice > 0; s_slice--) { - const path_part = deco.path.slice(0, s_slice).join("/"); - const path_wildcards = []; - if (s_slice === deco.path.length) { - if (deco.path_trailing_dash) - path_wildcards.push("/"); - path_wildcards.push(""); - } - if (s_slice === deco.path.length - 1 && - deco.path[s_slice] !== "*") - path_wildcards.push("/*"); - if (s_slice < deco.path.length && - (deco.path[s_slice] !== "**" || - s_slice < deco.path.length - 1)) - path_wildcards.push("/**"); - if (deco.path[s_slice] !== "***" || s_slice < deco.path.length) - path_wildcards.push("/***"); - - for (const path_wildcard of path_wildcards) { - const path_pattern = path_part + path_wildcard; - - const pattern = deco.proto + domain_pattern + path_pattern; - - if (callback(pattern) === false) - return; - } - } +function* each_path_pattern(deco) +{ + for (let slice = deco.path.length; slice >= 0; slice--) { + const path_part = ["", ...deco.path.slice(0, slice)].join("/"); + const path_wildcards = []; + if (slice === deco.path.length && !deco.path_truncated) { + if (deco.trailing_dash) + yield path_part + "/"; + if (slice > 0 || deco.proto !== "file") + yield path_part; } + if (slice === deco.path.length - 1 && !deco.path_truncated && + deco.path[slice] !== "*") + yield path_part + "/*"; + if (slice < deco.path.length - 1) + yield path_part + "/**"; + if (slice !== deco.path.length - 1 || deco.path_truncated || + deco.path[slice] !== "***") + yield path_part + "/***"; } } -function possible_patterns(url) +/* Generate every possible pattern that matches url. */ +function* each_url_pattern(url) { - const patterns = []; - for_each_possible_pattern(url, patterns.push); + const deco = deconstruct_url(url); - return patterns; + if (deco === undefined) { + console.error("bad url format", url); + return false; + } + + const all_domains = deco.domain ? each_domain_pattern(deco) : [""]; + for (const domain of all_domains) { + for (const path of each_path_pattern(deco)) + yield `${deco.proto}://${domain}${path}`; + } } /* * EXPORTS_START - * EXPORT url_matches - * EXPORT for_each_possible_pattern - * EXPORT possible_patterns + * EXPORT each_url_pattern + * EXPORT deconstruct_url * EXPORTS_END */ diff --git a/common/sanitize_JSON.js b/common/sanitize_JSON.js index 8b86d2d..4cf1ef4 100644 --- a/common/sanitize_JSON.js +++ b/common/sanitize_JSON.js @@ -1,6 +1,7 @@ /** - * part of Hachette - * Powerful, full-blown format enforcer for externally-obtained JSON + * This file is part of Haketilo. + * + * Function: Powerful, full-blown format enforcer for externally-obtained JSON. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. diff --git a/common/settings_query.js b/common/settings_query.js index e85ae63..7e1315e 100644 --- a/common/settings_query.js +++ b/common/settings_query.js @@ -1,5 +1,7 @@ /** - * Hachette querying page settings with regard to wildcard records + * This file is part of Haketilo. + * + * Function: Querying page settings. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. @@ -8,30 +10,25 @@ /* * IMPORTS_START * IMPORT TYPE_PREFIX - * IMPORT for_each_possible_pattern + * IMPORT each_url_pattern * IMPORTS_END */ -function check_pattern(storage, pattern, multiple, matched) -{ - const settings = storage.get(TYPE_PREFIX.PAGE, pattern); - - if (settings === undefined) - return; - - matched.push([pattern, settings]); - - if (!multiple) - return false; -} - function query(storage, url, multiple) { const matched = []; const cb = p => check_pattern(storage, p, multiple, matched); - for_each_possible_pattern(url, cb); + for (const pattern of each_url_pattern(url)) { + const result = [pattern, storage.get(TYPE_PREFIX.PAGE, pattern)]; + if (result[1] === undefined) + continue; + + if (!multiple) + return result; + matched.push(result); + } - return multiple ? matched : (matched[0] || [undefined, undefined]); + return multiple ? matched : [undefined, undefined]; } function query_best(storage, url) diff --git a/common/storage_client.js b/common/storage_client.js index 2b2f495..ef4a0b8 100644 --- a/common/storage_client.js +++ b/common/storage_client.js @@ -1,5 +1,7 @@ /** - * Hachette storage through connection (client side) + * This file is part of Haketilo. + * + * Function: Storage through messages (client side). * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. diff --git a/common/storage_light.js b/common/storage_light.js new file mode 100644 index 0000000..246e5eb --- /dev/null +++ b/common/storage_light.js @@ -0,0 +1,131 @@ +/** + * This file is part of Haketilo. + * + * Function: Storage manager, lighter than the previous one. + * + * Copyright (C) 2021 Wojtek Kosior + * Redistribution terms are gathered in the `copyright' file. + */ + +/* + * IMPORTS_START + * IMPORT TYPE_PREFIX + * IMPORT raw_storage + * IMPORT is_mozilla + * IMPORT observables + * IMPORTS_END + */ + +const reg_spec = new Set(["\\", "[", "]", "(", ")", "{", "}", ".", "*", "+"]); +const escape_reg_special = c => reg_spec.has(c) ? "\\" + c : c; + +function make_regex(name) +{ + return new RegExp(`^${name.split("").map(escape_reg_special).join("")}\$`); +} + +const listeners_by_callback = new Map(); + +function listen(callback, prefix, name) +{ + let by_prefix = listeners_by_callback.get(callback); + if (!by_prefix) { + by_prefix = new Map(); + listeners_by_callback.set(callback, by_prefix); + } + + let by_name = by_prefix.get(prefix); + if (!by_name) { + by_name = new Map(); + by_prefix.set(prefix, by_name); + } + + let name_reg = by_name.get(name); + if (!name_reg) { + name_reg = name.test ? name : make_regex(name); + by_name.set(name, name_reg); + } +} + +function no_listen(callback, prefix, name) +{ + const by_prefix = listeners_by_callback.get(callback); + if (!by_prefix) + return; + + const by_name = by_prefix.get(prefix); + if (!by_name) + return; + + const name_reg = by_name.get(name); + if (!name_reg) + return; + + by_name.delete(name); + + if (by_name.size === 0) + by_prefix.delete(prefix); + + if (by_prefix.size === 0) + listeners_by_callback.delete(callback); +} + +function storage_change_callback(changes, area) +{ + if (is_mozilla && area !== "local") + {console.log("area", area);return;} + + for (const item of Object.keys(changes)) { + for (const [callback, by_prefix] of listeners_by_callback.entries()) { + const by_name = by_prefix.get(item[0]); + if (!by_name) + continue; + + for (const reg of by_name.values()) { + if (!reg.test(item.substring(1))) + continue; + + try { + callback(item, changes[item]); + } catch(e) { + console.error(e); + } + } + } + } +} + +raw_storage.listen(storage_change_callback); + + +const created_observables = new Map(); + +async function observe(prefix, name) +{ + const observable = observables.make(); + const callback = (it, ch) => observables.set(observable, ch.newValue); + listen(callback, prefix, name); + created_observables.set(observable, [callback, prefix, name]); + observables.silent_set(observable, await raw_storage.get(prefix + name)); + + return observable; +} + +const observe_var = name => observe(TYPE_PREFIX.VAR, name); + +function no_observe(observable) +{ + no_listen(...created_observables.get(observable) || []); + created_observables.delete(observable); +} + +const light_storage = {}; +Object.assign(light_storage, raw_storage); +Object.assign(light_storage, + {listen, no_listen, observe, observe_var, no_observe}); + +/* + * EXPORTS_START + * EXPORT light_storage + * EXPORTS_END + */ diff --git a/common/storage_raw.js b/common/storage_raw.js new file mode 100644 index 0000000..e354b6b --- /dev/null +++ b/common/storage_raw.js @@ -0,0 +1,55 @@ +/** + * This file is part of Haketilo. + * + * Function: Basic wrappers for storage API functions. + * + * Copyright (C) 2021 Wojtek Kosior + * Redistribution terms are gathered in the `copyright' file. + */ + +/* + * IMPORTS_START + * IMPORT TYPE_PREFIX + * IMPORT browser + * IMPORT is_chrome + * IMPORTS_END + */ + +async function get(key) +{ + /* Fix for fact that Chrome does not use promises here */ + const promise = is_chrome ? + new Promise(resolve => chrome.storage.local.get(key, resolve)) : + browser.storage.local.get(key); + + return (await promise)[key]; +} + +async function set(key_or_object, value) +{ + const arg = typeof key_or_object === "object" ? + key_or_object : {[key_or_object]: value}; + return browser.storage.local.set(arg); +} + +async function set_var(name, value) +{ + return set(TYPE_PREFIX.VAR + name, value); +} + +async function get_var(name) +{ + return get(TYPE_PREFIX.VAR + name); +} + +const on_changed = browser.storage.onChanged || browser.storage.local.onChanged; +const listen = cb => on_changed.addListener(cb); +const no_listen = cb => on_changed.removeListener(cb); + +const raw_storage = {get, set, get_var, set_var, listen, no_listen}; + +/* + * EXPORTS_START + * EXPORT raw_storage + * EXPORTS_END + */ diff --git a/common/stored_types.js b/common/stored_types.js index bfceba6..a693b1c 100644 --- a/common/stored_types.js +++ b/common/stored_types.js @@ -1,5 +1,7 @@ /** - * Hachette stored item types "enum" + * This file is part of Haketilo. + * + * Function: Define an "enum" of stored item types. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. diff --git a/compute_scripts.awk b/compute_scripts.awk new file mode 100644 index 0000000..1f3b11e --- /dev/null +++ b/compute_scripts.awk @@ -0,0 +1,205 @@ +# SPDX-License-Identifier: CC0-1.0 +# +# Process javascript files and resolve dependencies between them +# +# This file is part of Haketilo +# +# Copyright (C) 2021, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +function read_file(filename, + imports_state, exports_state, line, record, result) { + imports_state = "not_started" + exports_state = "not_started" + + do { + result = (getline line < filename) + if (result < 0) { + printf "error reading %s", filename + exit 1 + } + + if (imports_state == "started" && + line ~ /^([[:space:]]*\*[[:space:]]+)?IMPORT[[:space:]]+[_a-zA-Z][_a-zA-Z0-9]*[[:space:]]*$/) { + record = line + + sub(/^([[:space:]]*\*[[:space:]]+)?IMPORT[[:space:]]+/, "", record) + sub(/([[:space:]]+$)/, "", record) + + imports[filename,++import_counts[filename]] = record + } + if (imports_state == "started" && + line ~ /^([[:space:]]*\*[[:space:]]+)?IMPORTS_END[[:space:]]*$/) + imports_state = "finished" + if (imports_state == "not_started" && + line ~ /^([[:space:]]*\*[[:space:]]+)?IMPORTS_START[[:space:]]*$/) + imports_state = "started" + + if (exports_state == "started" && + line ~ /^([[:space:]]*\*[[:space:]]+)?EXPORT[[:space:]]+[_a-zA-Z][_a-zA-Z0-9]*[[:space:]]*$/) { + record = line + + sub(/^([[:space:]]*\*[[:space:]]+)?EXPORT[[:space:]]+/, "", record) + sub(/([[:space:]]+$)/, "", record) + + if (record in exports) { + printf "ERROR: '%s' exported by both %s and %s\n", + exports[record], filename > "/dev/stderr" + } + + provides[record] = filename + exports[filename,++export_counts[filename]] = record + } + if (exports_state == "started" && + line ~ /^([[:space:]]*\*[[:space:]]+)?EXPORTS_END[[:space:]]*$/) + exports_state = "finished" + if (exports_state == "not_started" && + line ~ /^([[:space:]]*\*[[:space:]]+)?EXPORTS_START[[:space:]]*$/) + exports_state = "started" + } while (result > 0) + + if (imports_state == "started") { + printf "ERROR: Unclosed IMPORTS list in '%s'\n", filename \ + > "/dev/stderr" + exit 1 + } + + if (exports_state == "started") { + printf "ERROR: Unclosed EXPORTS list in '%s'\n", filename \ + > "/dev/stderr" + exit 1 + } + + close(filename) +} + +function print_file(filename, line) { + while ((getline line < filename) > 0) + print(line) + + close(filename) +} + +function print_imports_code(filename, i, count, import_name) { + count = import_counts[filename] + for (i = 1; i <= count; i++) { + import_name = imports[filename,i] + printf "const %s = window.haketilo_exports.%s;\n", + import_name, import_name + } +} + +function print_exports_code(filename, i, count, export_name) { + count = export_counts[filename] + for (i = 1; i <= count; i++) { + export_name = exports[filename,i] + printf "window.haketilo_exports.%s = %s;\n", export_name, export_name + } +} + +function partially_wrap_file(filename) { + print_imports_code(filename) + printf "\n\n" + + print_file(filename) + + printf "\n\n" + print_exports_code(filename) +} + +function wrap_file(filename) { + print "\"use strict\";\n\n({fun: (function() {\n" + + partially_wrap_file(filename) + + print "\n})}).fun();" +} + +function compute_dependencies(filename, i, count, import_name, next_file) { + if (processed[filename] == "used") + return 0 + + if (processed[filename] == "on_stack") { + printf "import loop on %s\n", filename > "/dev/stderr" + return 1 + } + + processed[filename] = "on_stack" + + count = import_counts[filename] + for (i = 1; i <= count; i++) { + import_name = imports[filename,i] + if (!(import_name in provides)) { + printf "nothing exports %s, required by %s\n", + import_name, filename > "/dev/stderr" + return 1 + } + + if (compute_dependencies(provides[import_name]) > 0) { + printf "when satisfying %s for %s\n", + import_name, filename > "/dev/stderr" + return 1 + } + } + + processed[filename] = "used" + print filename + + return 0 +} + +function print_usage() { + printf "usage: %2 compute_scripts.awk script_dependencies|wrapped_code|partially_wrapped_code FILENAME[...]\n", + ARGV[0] > "/dev/stderr" + exit 1 +} + +function mock_exports_init() { + provides["browser"] = "exports_init.js" + provides["is_chrome"] = "exports_init.js" + provides["is_mozilla"] = "exports_init.js" + + processed["exports_init.js"] = "used" +} + +BEGIN { + operation = ARGV[1] + + if (ARGC < 3) + print_usage() + + root_filename = ARGV[2] + + for (i = 2; i < ARGC; i++) + filenames[ARGV[i]] + + mock_exports_init() + + for (filename in filenames) { + # A filename is allowed to appear multiple times in the list. + # Let's only process it once. + if (!(filename in processed)) + read_file(filename) + processed[filename] = "not_used" + } + + if (operation == "script_dependencies") { + print("exports_init.js") + if (compute_dependencies(root_filename) > 0) + exit 1 + } else if (operation == "partially_wrapped_code") { + partially_wrap_file(root_filename) + } else if (operation == "wrapped_code") { + wrap_file(root_filename) + } else { + print_usage() + } +} diff --git a/configure b/configure new file mode 100755 index 0000000..06a43eb --- /dev/null +++ b/configure @@ -0,0 +1,109 @@ +#!/bin/sh + +# This file is part of Haketilo +# +# Copyright (C) 2021, jahoti +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +set -e + +BROWSERPATH='' +SRCDIR='' +TARGET='' + +# Parse command line options +while [ "x$1" != x ]; do + case "$1" in + --srcdir=*) SRCDIR="$(echo "$1" | cut -c 10-)";; + --srcdir) SRCDIR="$2"; shift;; + "DESTDIR"=*) DESTDIR="$(echo "$1" | cut -c 9-)";; + "UPDATE_URL"=*) UPDATE_URL="$(echo "$1" | cut -c 12-)";; + --host=*) TARGET="$(echo "$1" | cut -c 8-)";; + --host) TARGET="$2"; shift;; + + # browsers + chromium | chrome | google-chrome | mozilla |\ + firefox | librewolf | icecat | iceweasel | abrowser |\ + iceweasel-uxp | tor-browser) TARGET=$1;; + *) echo Ignoring option "'$1'";; + esac + shift +done + +# Autodetect srcdir +if [ "x$SRCDIR" = x ]; then + SRCDIR=.. + if [ -f manifest.json ] && [ -f write_makefile.sh ]; then + SRCDIR=. + fi +fi + +# Check srcdir +if [ ! -f "$SRCDIR"/manifest.json ]; then + echo Invalid source directory "'$SRCDIR'": missing manifest.json >&2 + exit 1 +elif [ ! -f "$SRCDIR"/write_makefile.sh ]; then + echo Invalid source directory "'$SRCDIR'": missing write_makefile.sh >&2 + exit 1 +fi + +# Autodetect target +if [ "x$TARGET" = x ]; then + echo Detecting target automatically. + if [ -h /etc/alternatives/x-www-browser ]; then + BROWSERPATH="$(realpath /etc/alternatives/x-www-browser)" + TARGET="$(/etc/alternatives/x-www-browser --version 2> /dev/null | + tail -n 1 | awk '{ print $1 }' | tr [A-Z] [a-z])" + else + echo Warning: could not find target automatically. >&2 + echo Some make rules may fail. >&2 + fi +else + BROWSERPATH="$(realpath "$(which $TARGET)")" +fi + +# Check and standardize target +case "$TARGET" in + mozilla | firefox | abrowser | icecat | iceweasel-uxp |\ + librewolf | iceweasel | gnu | tor-browser) TARGET=mozilla;; + chromium | chrome | google-chrome | google) TARGET=chromium;; + "") ;; + *) echo Invalid target "'$TARGET'" >&2; exit 2;; +esac + +# Autodetect DESTDIR (no check needed) +if [ "x$DESTDIR" = x ]; then + echo Guessing installation directory. + if [ -n "$BROWSERPATH" ] && [ -n "$TARGET" ]; then + DESTDIR="$(dirname "$BROWSERPATH")" # TODO: a hack for Debian? + if [ $TARGET = mozilla ]; then + DESTDIR="$DESTDIR"/browser + fi + DESTDIR="$DESTDIR"/extensions + else + echo Warning: could not guess installation directory. >&2 + echo Some make rules may fail. >&2 + fi +fi + +# Write record.conf (LEAVE SRCDIR FIRST) +echo srcdir = "$SRCDIR" > record.conf +echo default_target = "$TARGET" >> record.conf +echo DESTDIR = "$DESTDIR" >> record.conf +echo UPDATE_URL = "$UPDATE_URL" >> record.conf + + +# Prepare and run write_makefile.sh (as config.status) +if [ ! -e config.status ]; then + cp "$SRCDIR"/write_makefile.sh config.status +fi + +./config.status diff --git a/content/activity_info_server.js b/content/activity_info_server.js index 81a25fb..aa92b75 100644 --- a/content/activity_info_server.js +++ b/content/activity_info_server.js @@ -1,7 +1,8 @@ /** - * part of Hachette - * Informing about activities performed by content script (script injection, - * script blocking). + * This file is part of Haketilo. + * + * Function: Informing the popup about what happens in the content script + * (script injection, script blocking, etc.). * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. @@ -41,7 +42,14 @@ function report_script(script_data) function report_settings(settings) { - report_activity("settings", settings); + const settings_clone = {}; + Object.assign(settings_clone, settings) + report_activity("settings", settings_clone); +} + +function report_document_type(is_html) +{ + report_activity("is_html", is_html); } function report_repo_query_action(update, port) @@ -91,5 +99,6 @@ function start_activity_info_server() * EXPORT start_activity_info_server * EXPORT report_script * EXPORT report_settings + * EXPORT report_document_type * EXPORTS_END */ diff --git a/content/freezer.js b/content/freezer.js deleted file mode 100644 index 9dbc95e..0000000 --- a/content/freezer.js +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Helper functions for blocking scripts in pages, based off NoScript's lib/DocumentFreezer.js - * - * Copyright (C) 2005-2021 Giorgio Maone - https://maone.net - * Copyright (C) 2021 jahoti - * Redistribution terms are gathered in the `copyright' file. - */ - -const loaderAttributes = ["href", "src", "data"]; -const jsOrDataUrlRx = /^(?:data:(?:[^,;]*ml|unknown-content-type)|javascript:)/i; - -function sanitize_attributes(element) { - if (element._frozen) - return; - let fa = []; - let loaders = []; - let attributes = element.attributes || []; - - for (let a of attributes) { - let name = a.localName.toLowerCase(); - if (loaderAttributes.includes(name)) - if (jsOrDataUrlRx.test(a.value)) - loaders.push(a); - - else if (name.startsWith("on")) { - console.debug("Removing", a, element.outerHTML); - fa.push(a.cloneNode()); - a.value = ""; - element[name] = null; - } - } - if (loaders.length) { - for (let a of loaders) { - fa.push(a.cloneNode()); - a.value = "javascript://frozen"; - } - if ("contentWindow" in element) - element.replaceWith(element = element.cloneNode(true)); - - } - if (fa.length) - element._frozenAttributes = fa; - element._frozen = true; -} - -function mozilla_suppress_scripts(e) { - if (document.readyState === 'complete') { - removeEventListener('beforescriptexecute', blockExecute, true); - console.log('Script suppressor has detached.'); - return; - } - if (e.isTrusted && !e.target._hachette_payload) { - e.preventDefault(); - console.log('Suppressed script', e.target); - } -}; - -/* - * EXPORTS_START - * EXPORT mozilla_suppress_scripts - * EXPORT sanitize_attributes - * EXPORTS_END - */ diff --git a/content/main.js b/content/main.js index 9ed557c..ce1ff7a 100644 --- a/content/main.js +++ b/content/main.js @@ -1,5 +1,7 @@ /** - * Hachette main content script run in all frames + * This file is part of Haketilo. + * + * Function: Main content script that runs in all frames. * * Copyright (C) 2021 Wojtek Kosior * Copyright (C) 2021 jahoti @@ -9,123 +11,310 @@ /* * IMPORTS_START * IMPORT handle_page_actions - * IMPORT url_extract_target - * IMPORT gen_unique * IMPORT gen_nonce - * IMPORT csp_rule * IMPORT is_privileged_url - * IMPORT sanitize_attributes - * IMPORT mozilla_suppress_scripts + * IMPORT browser * IMPORT is_chrome * IMPORT is_mozilla * IMPORT start_activity_info_server + * IMPORT make_csp_rule + * IMPORT csp_header_regex + * IMPORT report_settings * IMPORTS_END */ +document.content_loaded = document.readyState === "complete"; +const wait_loaded = e => e.content_loaded ? Promise.resolve() : + new Promise(c => e.addEventListener("DOMContentLoaded", c, {once: true})); + +wait_loaded(document).then(() => document.content_loaded = true); + /* - * Due to some technical limitations the chosen method of whitelisting sites - * is to smuggle whitelist indicator in page's url as a "magical" string - * after '#'. Right now this is only supplemental in HTTP(s) pages where - * blocking of native scripts also happens through CSP header injection but is - * necessary for protocols like ftp:// and file://. - * - * The code that actually injects the magical string into ftp:// and file:// - * urls has not yet been added to the extension. + * In the case of HTML documents: + * 1. When injecting some payload we need to sanitize <meta> CSP tags before + * they reach the document. + * 2. Only <meta> tags inside <head> are considered valid by the browser and + * need to be considered. + * 3. We want to detach <html> from document, wait until its <head> completes + * loading, sanitize it and re-attach <html>. + * 4. We shall wait for anything to appear in or after <body> and take that as + * a sign <head> has finished loading. + * 5. Otherwise, getting the `DOMContentLoaded' event on the document shall also + * be a sign that <head> is fully loaded. */ -var nonce = undefined; +function make_body_start_observer(DOM_element, waiting) +{ + const observer = new MutationObserver(() => try_body_started(waiting)); + observer.observe(DOM_element, {childList: true}); + return observer; +} -function handle_mutation(mutations, observer) +function try_body_started(waiting) { - if (document.readyState === 'complete') { - console.log("mutation handling complete"); - observer.disconnect(); - return; - } - for (const mutation of mutations) { - for (const node of mutation.addedNodes) - block_node(node); + const body = waiting.detached_html.querySelector("body"); + + if ((body && (body.firstChild || body.nextSibling)) || + waiting.doc.documentElement.nextSibling) { + finish_waiting(waiting); + return true; } + + if (body && waiting.observers.length < 2) + waiting.observers.push(make_body_start_observer(body, waiting)); } -function block_nodes_recursively(node) +function finish_waiting(waiting) { - block_node(node); - for (const child of node.children) - block_nodes_recursively(child); + if (waiting.finished) + return; + waiting.finished = true; + waiting.observers.forEach(observer => observer.disconnect()); + setTimeout(waiting.callback, 0); } -function block_node(node) +function _wait_for_head(doc, detached_html, callback) { + const waiting = {doc, detached_html, callback, observers: []}; + + if (try_body_started(waiting)) + return; + + waiting.observers = [make_body_start_observer(detached_html, waiting)]; + + wait_loaded(doc).then(() => finish_waiting(waiting)); +} + +function wait_for_head(doc, detached_html) +{ + return new Promise(cb => _wait_for_head(doc, detached_html, cb)); +} + +const blocked_str = "blocked"; + +function block_attribute(node, attr, ns=null) +{ + const [hasa, geta, seta, rema] = ["has", "get", "set", "remove"] + .map(m => (n, ...args) => typeof ns === "string" ? + n[`${m}AttributeNS`](ns, ...args) : n[`${m}Attribute`](...args)); /* - * Modifying <script> element doesn't always prevent its execution in some - * Mozilla browsers. This is Chromium-specific code. + * Disabling attributes by prepending `-blocked' allows them to still be + * relatively easily accessed in case they contain some useful data. */ - if (node.tagName === "SCRIPT") { - block_script(node); - return; + const construct_name = [attr]; + while (hasa(node, construct_name.join(""))) + construct_name.unshift(blocked_str); + + while (construct_name.length > 1) { + construct_name.shift(); + const name = construct_name.join(""); + seta(node, `${blocked_str}-${name}`, geta(node, name)); + } + + rema(node, attr); +} + +/* + * Used to disable `<script>'s and `<meta>'s that have not yet been added to + * live DOM (doesn't work for those already added). + */ +function sanitize_meta(meta) +{ + if (csp_header_regex.test(meta.httpEquiv) && meta.content) + block_attribute(meta, "content"); +} + +function sanitize_script(script) +{ + script.haketilo_blocked_type = script.getAttribute("type"); + script.type = "text/plain"; +} + +/* + * Executed after `<script>' has been connected to the DOM, when it is no longer + * eligible for being executed by the browser. + */ +function desanitize_script(script) +{ + script.setAttribute("type", script.haketilo_blocked_type); + + if ([null, undefined].includes(script.haketilo_blocked_type)) + script.removeAttribute("type"); + + delete script.haketilo_blocked_type; +} + +const bad_url_reg = /^data:([^,;]*ml|unknown-content-type)/i; +function sanitize_urls(element) +{ + for (const attr of [...element.attributes || []] + .filter(attr => /^(href|src|data)$/i.test(attr.localName)) + .filter(attr => bad_url_reg.test(attr.value))) + block_attribute(element, attr.localName, attr.namespaceURI); +} + +function start_data_urls_sanitizing(doc) +{ + doc.querySelectorAll("*[href], *[src], *[data]").forEach(sanitize_urls); + if (!doc.content_loaded) { + const mutation_handler = m => m.addedNodes.forEach(sanitize_urls); + const mo = new MutationObserver(ms => ms.forEach(mutation_handler)); + mo.observe(doc, {childList: true, subtree: true}); + wait_loaded(doc).then(() => mo.disconnect()); } +} + +/* + * Normally, we block scripts with CSP. However, Mozilla does optimizations that + * cause part of the DOM to be loaded when our content scripts get to run. Thus, + * before the CSP rules we inject (for non-HTTP pages) become effective, we need + * to somehow block the execution of `<script>'s and intrinsics that were + * already there. Additionally, some browsers (IceCat 60) seem to have problems + * applying this CSP to non-inline `<scripts>' in certain scenarios. + */ +function prevent_script_execution(event) +{ + if (!event.target.haketilo_payload) + event.preventDefault(); +} - sanitize_attributes(node); +function mozilla_initial_block(doc) +{ + doc.addEventListener("beforescriptexecute", prevent_script_execution); - if (node.tagName === "HEAD") - inject_csp(node); + for (const elem of doc.querySelectorAll("*")) { + [...elem.attributes].map(attr => attr.localName) + .filter(attr => /^on/.test(attr) && elem.wrappedJSObject[attr]) + .forEach(attr => elem.wrappedJSObject[attr] = null); + } } -function block_script(node) +/* + * Here we block all scripts of a document which might be either and + * HTMLDocument or an XMLDocument. Modifying an XML document might disrupt + * Mozilla's XML preview. This is an unfortunate thing we have to accept for + * now. XML documents *have to* be sanitized as well because they might + * contain `<script>' tags (or on* attributes) with namespace declared as + * "http://www.w3.org/1999/xhtml" or "http://www.w3.org/2000/svg" which allows + * javascript execution. + */ +async function sanitize_document(doc, policy) { /* - * Disabling scripts this way allows them to still be relatively - * easily accessed in case they contain some useful data. + * Blocking of scripts that are in the DOM from the beginning. Needed for + * Mozilla. */ - if (node.hasAttribute("type")) - node.setAttribute("blocked-type", node.getAttribute("type")); - node.setAttribute("type", "application/json"); + if (is_mozilla) + mozilla_initial_block(doc); + + /* + * Ensure our CSP rules are employed from the beginning. This CSP injection + * method is, when possible, going to be applied together with CSP rules + * injected using webRequest. + * Using elements namespaced as HTML makes this CSP injection also work for + * non-HTML documents. + */ + const html = new DOMParser().parseFromString(`<html><head><meta \ +http-equiv="Content-Security-Policy" content="${make_csp_rule(policy)}"\ +/></head><body>Loading...</body></html>`, "text/html").documentElement; + + /* + * Root node gets hijacked now, to be re-attached after <head> is loaded + * and sanitized. + */ + const root = doc.documentElement; + root.replaceWith(html); + + /* + * When we don't inject payload, we neither block document's CSP `<meta>' + * tags nor wait for `<head>' to be parsed. + */ + if (policy.has_payload) { + await wait_for_head(doc, root); + + root.querySelectorAll("head meta") + .forEach(m => sanitize_meta(m, policy)); + } + + root.querySelectorAll("script").forEach(s => sanitize_script(s, policy)); + html.replaceWith(root); + root.querySelectorAll("script").forEach(s => desanitize_script(s, policy)); + + start_data_urls_sanitizing(doc); } -function inject_csp(head) +async function _disable_service_workers() { - console.log('injecting CSP'); + if (!navigator.serviceWorker) + return; - let meta = document.createElement("meta"); - meta.setAttribute("http-equiv", "Content-Security-Policy"); - meta.setAttribute("content", csp_rule(nonce)); + const registrations = await navigator.serviceWorker.getRegistrations(); + if (registrations.length === 0) + return; + + console.warn("Service Workers detected on this page! Unregistering and reloading."); - if (head.firstElementChild === null) - head.appendChild(meta); - else - head.insertBefore(meta, head.firstElementChild); + try { + await Promise.all(registrations.map(r => r.unregister())); + } finally { + location.reload(); + } + + /* Never actually return! */ + return new Promise(() => 0); } -if (!is_privileged_url(document.URL)) { - const targets = url_extract_target(document.URL); - if (targets.policy) { - if (targets.target2) - window.location.href = targets.base_url + targets.target2; - else - history.replaceState(null, "", targets.base_url); +/* + * Trying to use servce workers APIs might result in exceptions, for example + * when in a non-HTML document. Because of this, we wrap the function that does + * the actual work in a try {} block. + */ +async function disable_service_workers() +{ + try { + await _disable_service_workers() + } catch (e) { + console.debug("Exception thrown during an attempt to detect and disable service workers.", e); } +} - const policy = targets.current ? targets.policy : {}; +function synchronously_get_policy(url) +{ + const encoded_url = encodeURIComponent(url); + const request_url = `${browser.runtime.getURL("dummy")}?url=${encoded_url}`; - nonce = policy.nonce || gen_nonce(); - handle_page_actions(nonce); + try { + var xhttp = new XMLHttpRequest(); + xhttp.open("GET", request_url, false); + xhttp.send(); + } catch(e) { + console.error("Failure to synchronously fetch policy for url.", e); + return {allow: false}; + } - if (!policy.allow) { - block_nodes_recursively(document.documentElement); + const policy = /^[^?]*\?settings=(.*)$/.exec(xhttp.responseURL)[1]; + return JSON.parse(decodeURIComponent(policy)); +} - if (is_chrome) { - var observer = new MutationObserver(handle_mutation); - observer.observe(document.documentElement, { - attributes: true, - childList: true, - subtree: true - }); - } +if (!is_privileged_url(document.URL)) { + const policy = synchronously_get_policy(document.URL); - if (is_mozilla) - addEventListener('beforescriptexecute', mozilla_suppress_scripts, true); - } + if (!(document instanceof HTMLDocument)) + delete policy.payload; + + console.debug("current policy", policy); + + report_settings(policy); + + policy.nonce = gen_nonce(); + + const doc_ready = Promise.all([ + policy.allow ? Promise.resolve() : sanitize_document(document, policy), + policy.allow ? Promise.resolve() : disable_service_workers(), + wait_loaded(document) + ]); + + handle_page_actions(policy, doc_ready); start_activity_info_server(); } diff --git a/content/page_actions.js b/content/page_actions.js index aff56b8..845e452 100644 --- a/content/page_actions.js +++ b/content/page_actions.js @@ -1,5 +1,7 @@ /** - * Hachette handling of page actions in content scripts + * This file is part of Haketilo. + * + * Function: Handle page actions in a content script. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. @@ -10,14 +12,17 @@ * IMPORT CONNECTION_TYPE * IMPORT browser * IMPORT report_script - * IMPORT report_settings + * IMPORT report_document_type * IMPORTS_END */ -var port; -var loaded = false; -var scripts_awaiting = []; -var nonce; +let policy; +/* Snapshot url and content type early; these can be changed by other code. */ +let url; +let is_html; +let port; +let loaded = false; +let scripts_awaiting = []; function handle_message(message) { @@ -31,11 +36,12 @@ function handle_message(message) scripts_awaiting.push(script_text); } } - if (action === "settings") - report_settings(data); + else { + console.error(`Bad page action '${action}'.`); + } } -function document_loaded(event) +function document_ready(event) { loaded = true; @@ -47,22 +53,32 @@ function document_loaded(event) function add_script(script_text) { + if (!is_html) + return; + let script = document.createElement("script"); script.textContent = script_text; - script.setAttribute("nonce", nonce); - script._hachette_payload = true; + script.setAttribute("nonce", policy.nonce); + script.haketilo_payload = true; document.body.appendChild(script); report_script(script_text); } -function handle_page_actions(script_nonce) { - document.addEventListener("DOMContentLoaded", document_loaded); - port = browser.runtime.connect({name : CONNECTION_TYPE.PAGE_ACTIONS}); - port.onMessage.addListener(handle_message); - port.postMessage({url: document.URL}); +function handle_page_actions(_policy, doc_ready_promise) { + policy = _policy; - nonce = script_nonce; + url = document.URL; + is_html = document instanceof HTMLDocument; + report_document_type(is_html); + + doc_ready_promise.then(document_ready); + + if (policy.payload) { + port = browser.runtime.connect({name : CONNECTION_TYPE.PAGE_ACTIONS}); + port.onMessage.addListener(handle_message); + port.postMessage({payload: policy.payload}); + } } /* diff --git a/content/repo_query.js b/content/repo_query.js index 3708108..637282c 100644 --- a/content/repo_query.js +++ b/content/repo_query.js @@ -1,6 +1,7 @@ /** - * part of Hachette - * Getting available content for site from remote repositories. + * This file is part of Haketilo. + * + * Function: Getting available content for site from remote repositories. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. @@ -1,12 +1,13 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: Hachette +Upstream-Name: Haketilo Source: https://git.koszko.org/browser-extension/ Files: * Copyright: 2021 Wojtek Kosior <koszko@koszko.org> License: GPL-3+-javascript or Alicense-1.0 -Files: *.sh +Files: *.sh default_settings.json Makefile.in compute_scripts.awk + CHROMIUM_exports_init.js Copyright: 2021 Wojtek Kosior <koszko@koszko.org> 2021 jahoti <jahoti@tilde.team> License: CC0 @@ -20,15 +21,31 @@ Copyright: 2021 Wojtek Kosior <koszko@koszko.org> 2021 jahoti <jahoti@tilde.team> License: GPL-3+-javascript or Alicense-1.0 +Files: background/stream_filter.js +Copyright: 2018 Giorgio Maone <giorgio@maone.net> + 2021 Wojtek Kosior <koszko@koszko.org> +License: GPL-3+-javascript or Alicense-1.0, and GPL-3+ +Comment: Code by Wojtek is dual-licensed under GPL-3+-javascript and + Alicense-1.0. Giorgio's code is under GPL-3+. + Files: *.html README.txt copyright Copyright: 2021 Wojtek Kosior <koszko@koszko.org> License: GPL-3+ or Alicense-1.0 or CC-BY-SA-4.0 +Files: html/*.css +Copyright: 2021 Wojtek Kosior <koszko@koszko.org> +License: GPL-3+ or Alicense-1.0 or CC-BY-SA-4.0 + Files: html/base.css Copyright: 2021 Wojtek Kosior <koszko@koszko.org> 2021 Nicholas Johnson <nicholasjohnson@posteo.org> License: GPL-3+ or Alicense-1.0 or CC-BY-SA-4.0 +Files: html/reset.css +Copyright: 2008,2011 Eric A. Meyer +License: public-domain + This stylesheet has been place in public domain by its author. + Files: common/sha256.js Copyright: 2014-2017 Chen, Yi-Cyuan <emn178@gmail.com> License: Expat @@ -51,18 +68,28 @@ License: Expat OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -Files: content/freezer.js -Copyright: 2005-2021 Giorgio Maone - https://maone.net - 2021 jahoti <jahoti@tilde.team> -License: GPL-2+ - Files: test/* Copyright: 2021 jahoti <jahoti@tilde.team> + 2021 Wojtek Kosior <koszko@koszko.org> License: AGPL-3+ +Comment: Wojtek Kosior promises not to sue even in case of violations + of the license. + +Files: test/__init__.py test/unit/* + test/default_profiles/icecat_empty/extensions.json +Copyright: 2021 Wojtek Kosior <koszko@koszko.org> +License: CC0 + +Files: test/profiles.py test/script_loader.py test/unit/conftest.py +Copyright: 2021 Wojtek Kosior <koszko@koszko.org> +License: GPL-3+ +Comment: Wojtek Kosior promises not to sue even in case of violations + of the license. -Files: test/proxy_core.py test/init.sh +Files: test/proxy_core.py Copyright: 2015, inaz2 2021 jahoti <jahoti@tilde.team> + 2021 Wojtek Kosior <koszko@koszko.org> License: BSD-3 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -74,9 +101,9 @@ License: BSD-3 this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. . - * Neither the name of proxy2 nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. + * Neither the name of proxy2 nor the names of its contributors may be used to + endorse or promote products derived from this software without specific prior + written permission. . THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE @@ -88,11 +115,8 @@ License: BSD-3 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Files: test/gorilla.py test/misc_constants.py test/world_wide_library.py test/data/pages/* -Copyright: 2021 jahoti <jahoti@tilde.team> - 2021 jahoti <jahoti@tilde.team> -License: AGPL-3+ or Alicense-1.0 +Comment: Wojtek Kosior promises not to sue even in case of violations + of the license. Files: licenses/* Copyright: 2001, 2002, 2011-2013 Creative Commons @@ -167,12 +191,6 @@ License: GPL-2+ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -License: CC-BY-SA-4.0 - See `licenses/cc-by-sa-4.0.txt' - -License: CC-BY-4.0 - See `licenses/cc-by-4.0.txt' - License: AGPL-3+ This program is free software; you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as @@ -187,3 +205,9 @@ License: AGPL-3+ You should have received a copy of the GNU Affero General Public License along with this program; if not, you can get it from `https://www.gnu.org/licenses/agpl-3.0.html'. + +License: CC-BY-SA-4.0 + See `licenses/cc-by-sa-4.0.txt' + +License: CC-BY-4.0 + See `licenses/cc-by-4.0.txt' diff --git a/default_settings.json b/default_settings.json index 44fbca0..e7ea2c3 100644 --- a/default_settings.json +++ b/default_settings.json @@ -1,50 +1 @@ -[ - { - "sbandcamp": { - "text": "/*\n\tCopyright © 2021 jahoti (jahoti@tilde.team)\n\t\n\tLicensed under the Apache License, Version 2.0 (the \"License\");\n\tyou may not use this file except in compliance with the License.\n\tYou may obtain a copy of the License at\n\t\n\t http://www.apache.org/licenses/LICENSE-2.0\n\t\n\tUnless required by applicable law or agreed to in writing, software\n\tdistributed under the License is distributed on an \"AS IS\" BASIS,\n\tWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n\tSee the License for the specific language governing permissions and\n\tlimitations under the License.\n*/\n\nvar div, player, playerBox = document.querySelector('.inline_player');\nplayerBox.innerHTML = '';\n\nfor (var track of JSON.parse(document.querySelector('[data-tralbum]').dataset.tralbum).trackinfo) {\n\tdiv = document.createElement('div');\n\tplayer = document.createElement('audio');\n\t\n\tdiv.innerText = track.title + ': ';\n\tplayer.src = track.file['mp3-128']; // Is this always available?\n\tdiv.append(player);\n\tplayerBox.append(div);\n}" - } - }, - { - "sopencores": { - "text":"let data = JSON.parse(document.getElementById(\"__NEXT_DATA__\").textContent);\nlet sections = {};\nfor (let h1 of document.getElementsByClassName(\"cMJCrc\")) {\n let ul = document.createElement(\"ul\");\n if (h1.nextElementSibling !== null)\n\th1.parentNode.insertBefore(ul, h1.nextElementSibling);\n else\n\th1.parentNode.appendChild(ul);\n\n sections[h1.children[1].firstChild.textContent] = ul;\n}\n\nfor (let prop of data.props.pageProps.list) {\n let ul = sections[prop.category];\n if (ul === undefined) {\n\tconsole.log(`unknown category \"${prop.category}\" for project \"${prop.title}\"`);\n\tcontinue;\n }\n\n let li = document.createElement(\"li\");\n let a = document.createElement(\"a\");\n a.setAttribute(\"href\", \"/projects/\" + prop.slug);\n a.textContent = prop.title;\n\n li.appendChild(a);\n ul.appendChild(li);\n}\n" - } - }, - { - "ssumofus (sign petition)": { - "text": "/*\n\tCopyright © 2021 jahoti (jahoti@tilde.team)\n\t\n\tLicensed under the Apache License, Version 2.0 (the \"License\");\n\tyou may not use this file except in compliance with the License.\n\tYou may obtain a copy of the License at\n\t\n\t http://www.apache.org/licenses/LICENSE-2.0\n\t\n\tUnless required by applicable law or agreed to in writing, software\n\tdistributed under the License is distributed on an \"AS IS\" BASIS,\n\tWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n\tSee the License for the specific language governing permissions and\n\tlimitations under the License.\n*/\n\nfunction submitFormItem() {\n\tvar name, val, queryString = '', xhr = new content.XMLHttpRequest();\n\tfor (var formItem of this.querySelectorAll('select, input:not([type=\"radio\"]):not([type=\"checkbox\"])' +\n\t\t':not([type=\"submit\"]):not([type=\"reset\"])')) {\n\t\tqueryString += (queryString && '&') + formItem.name + '=' + encodeURIComponent(formItem.value);\n\t}\n\t\n\txhr.onreadystatechange = function () {\n\t\tif (this.readyState === 4) {\n\t\t\tif (this.status === 200) location.href = JSON.parse(this.responseText).follow_up_url;\n\t\t\telse if (this.status === 422) {\n\t\t\t\tvar failMessage = [], response = JSON.parse(this.responseText);\n\t\t\t\tfor (field in response.errors) for (error of response.errors[field]) {\n\t\t\t\t\tfailMessage.push('Field \"' + field + '\" ' + error);\n\t\t\t\t}\n\t\t\t\talert(failMessage.join('\\n'));\n\t\t\t}\n\t\t\telse alert('Submission failed: response code ' + this.status);\n\t\t}\n\t}\n\t\n\txhr.open('POST', this.action, true); // Manually add the domain, as it's not properly handled in extensions\n\txhr.setRequestHeader('X-CSRF-Token', csrf);\n\txhr.setRequestHeader('X-Requested-With', 'XMLHttpRequest');\n\txhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');\n\txhr.send(queryString);\n\treturn false;\n}\n\n// Apply CSS as necessary\nif (notice = document.querySelector('#petition-bar-main > span')) notice.style.display = 'none'; // Hide the totally mistaken (even without this extension) anti-anti-JS warning\ndocument.querySelector('.script-dependent').style.display = 'block';\ndocument.querySelector('.button-wrapper').style.position = 'static'; // Stop the \"submit\" button obscuring the form\n\n\n\ncsrf = document.querySelector('meta[name=\"csrf-token\"]').content\nfor (var button of document.querySelectorAll('button[type=\"submit\"].button.action-form__submit-button')) button.form.onsubmit = submitFormItem;" - } - }, - { - "sworldcat (library holdings)": { - "text": "/*\n\tCopyright © 2021 jahoti (jahoti@tilde.team)\n\t\n\tLicensed under the Apache License, Version 2.0 (the \"License\");\n\tyou may not use this file except in compliance with the License.\n\tYou may obtain a copy of the License at\n\t\n\t http://www.apache.org/licenses/LICENSE-2.0\n\t\n\tUnless required by applicable law or agreed to in writing, software\n\tdistributed under the License is distributed on an \"AS IS\" BASIS,\n\tWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n\tSee the License for the specific language governing permissions and\n\tlimitations under the License.\n*/\n\nvar pathParts = location.pathname.split('/'), itemRef = pathParts[pathParts.length - 1];\n\n// Generate a function which, when invoked, loads the catalog holdings starting at i (one-indexed) focused on loc\nfunction generateGoTo(i, set_loc) {\n\treturn function () {\n\t\t; // If this is a new search, \"set_loc\" won't be set; set it\n\t\tvar xhr = new content.XMLHttpRequest(), loc = set_loc || encodeURIComponent(locInput.value);\n\t\txhr.onreadystatechange = function () {\n\t\t\tif (this.readyState === 4) {\n\t\t\t\tif (this.status === 200) {\n\t\t\t\t\tretrieved.innerHTML = this.responseText;\n\t\t\t\t\t\n\t\t\t\t\tvar i, node = document.getElementById('libslocator');\n\t\t\t\t\tnode.parentNode.removeChild(node);\n\t\t\t\t\tfor (node of retrieved.querySelectorAll('a[href^=\"javascript:findLibs(\\'\\', \"]')) {\n\t\t\t\t\t\ti = parseInt(node.href.split(',', 2)[1]);\n\t\t\t\t\t\tnode.onclick = generateGoTo(i, loc);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\telse alert('Search failed: response code ' + this.status);\n\t\t\t}\n\t\t}\n\t\t\n\t\txhr.open('GET', 'https://www.worldcat.org/wcpa/servlet/org.oclc.lac.ui.ajax.ServiceServlet?wcoclcnum=' + itemRef + '&start_holding='\n\t\t\t\t+ i + '&serviceCommand=holdingsdata&loc=' + loc, true);\n\t\txhr.send();\n\t\treturn false; // Make sure the browser doesn't try to submit any holding form\n\t};\n}\n\n\nvar retriever = document.querySelector('.retrieving'), retrieved = document.getElementById('donelocator');\n\nvar locForm = document.createElement('form'), locLabel = document.createElement('label'), locInput = document.createElement('input'),\n\tlocSubmit = document.createElement('input');\n\nlocForm.appendChild(locLabel);\nlocForm.appendChild(locInput);\nlocForm.appendChild(locSubmit);\n\nlocInput.name = locLabel.htmlFor = 'cat_location';\nlocInput.type = 'text';\nlocInput.required = 'yes';\nlocLabel.innerText = 'Find copies closest to: ';\nlocSubmit.value = 'Go';\nlocSubmit.type = 'submit';\nlocForm.onsubmit = generateGoTo(1);\n\nretriever.parentNode.replaceChild(locForm, retriever);" - } - }, - { - "phttps://*.bandcamp.com/track/*": { - "components": ["s", "bandcamp"] - } - }, - { - "phttps://opencores.org/projects": { - "components": ["s", "opencores"] - } - }, - { - "phttps://actions.sumofus.org/a/*": { - "components": ["s", "sumofus (sign petition)"], - } - }, - { - "phttps://worldcat.org/title/**": { - "components": ["s", "worldcat (library holdings)"] - } - }, - { - "phttps://www.worldcat.org/title/**": { - "components": ["s", "worldcat (library holdings)"] - } - }, - { - "rhttps://api-demo.hachette-hydrilla.org": {} - } -] +[{"shaketilo demo script":{"url":"","hash":"","text":"/**\n * Haketilo demo script.\n *\n * Copyright (C) Wojtek Kosior\n * Available under the terms of Creative Commons Zero\n * <https://creativecommons.org/publicdomain/zero/1.0/legalcode>\n */\n\nconst banner = document.createElement(\"h2\");\n\nbanner.textContent = \"Hoooray! Haketilo works :D\"\n\nbanner.setAttribute(\"style\", `\\\nmargin: 1em; \\\nborder-radius: 1em 0px; \\\nbackground-color: #474; \\\npadding: 10px 20px; \\\ncolor: #eee; \\\nbox-shadow: 0 6px 8px 0 rgba(0,0,0,0.24), 0 17px 50px 0 rgba(0,0,0,0.19); \\\ndisplay: inline-block;\\\n`);\n\ndocument.body.prepend(banner);"}},{"bhaketilo demo bag":[["s","haketilo demo script"]]},{"phttps://hachette-hydrilla.org":{"components":["b","haketilo demo bag"],"allow":false}},{"rhttps://api-demo.hachette-hydrilla.org":{}}] diff --git a/html/DOM_helpers.js b/html/DOM_helpers.js index 2bff966..4fe118d 100644 --- a/html/DOM_helpers.js +++ b/html/DOM_helpers.js @@ -1,5 +1,7 @@ /** - * Hachette operations on DOM elements + * This file is part of Haketilo. + * + * Function: Operations on DOM elements. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. @@ -10,9 +12,27 @@ function by_id(id) return document.getElementById(id); } +const known_templates = new Map(); + +function get_template(template_id) +{ + let template = known_templates.get(template_id) || null; + if (template) + return template; + + for (const template_node of document.getElementsByTagName("TEMPLATE")) { + template = template_node.content.getElementById(template_id); + if (template) + break; + } + + known_templates.set(template_id, template); + return template; +} + function clone_template(template_id) { - const clone = document.getElementById(template_id).cloneNode(true); + const clone = get_template(template_id).cloneNode(true); const result_object = {}; const to_process = [clone]; @@ -24,7 +44,7 @@ function clone_template(template_id) result_object[template_key] = element; element.removeAttribute("id"); - element.removeAttribute("template_key"); + element.removeAttribute("data-template"); for (const child of element.children) to_process.push(child); @@ -36,6 +56,7 @@ function clone_template(template_id) /* * EXPORTS_START * EXPORT by_id + * EXPORT get_template * EXPORT clone_template * EXPORTS_END */ diff --git a/html/MOZILLA_scrollbar_fix.css b/html/MOZILLA_scrollbar_fix.css new file mode 100644 index 0000000..cdbd5c6 --- /dev/null +++ b/html/MOZILLA_scrollbar_fix.css @@ -0,0 +1,48 @@ +/** + * This file is part of Haketilo. + * + * Function: Hacky fix for vertical scrollbar width being included in child's + * width. + * + * Copyright (C) 2021 Wojtek Kosior + * Redistribution terms are gathered in the `copyright' file. + */ + +/* + * Under Mozilla browsers to avoid vertical scrollbar forcing horizontal + * scrollbar to appear in an element we add the `firefox_scrollbars_hacky_fix' + * class to an element for which width has to be reserved. + * + * This is a bit hacky and relies on some assumed width of Firefox scrollbar, I + * know. And must be excluded from Chromium builds. + * + * I came up with this hack when working on popup. Before that I had the + * scrollbar issue with tables in the options page and gave up there and made + * the scrollbal always visible. Now we could try applying this "fix" there, + * too! + */ + +.firefox_scrollbars_hacky_fix { + font-size: 0; +} + +.firefox_scrollbars_hacky_fix>div { + display: inline-block; + width: -moz-available; +} + +.firefox_scrollbars_hacky_fix>*>* { + font-size: initial; +} + +.firefox_scrollbars_hacky_fix::after { + content: ""; + display: inline-block; + visibility: hidden; + font-size: initial; + width: 14px; +} + +.firefox_scrollbars_hacky_fix.has_inline_content::after { + width: calc(14px - 0.3em); +} diff --git a/html/back_button.css b/html/back_button.css new file mode 100644 index 0000000..b83e834 --- /dev/null +++ b/html/back_button.css @@ -0,0 +1,51 @@ +/** + * This file is part of Haketilo. + * + * Function: Style for a "back" button with a CSS arrow image. + * + * Copyright (C) 2021 Wojtek Kosior + * Redistribution terms are gathered in the `copyright' file. + */ + +.back_button { + display: block; + width: auto; + height: auto; + background-color: white; + border: solid #454 0.4em; + border-left: none; + border-radius: 0 1.5em 1.5em 0; + cursor: pointer; +} + +.back_button:hover { + box-shadow: 0 6px 8px 0 rgba(0,0,0,0.24), 0 17px 50px 0 rgba(0,0,0,0.19); +} + +.back_button>div, .back_arrow { + width: 2em; + height: 0.5em; + background-color: #4CAF50; + border-radius: 0.3em; + margin: 1.15em 0.4em; +} + +.back_button>div::after, .back_arrow::after, +.back_button>div::before, .back_arrow::before { + content: ""; + display: block; + position: relative; + background-color: inherit; + width: 1.3em; + height: 0.5em; + transform: rotate(45deg); + border-radius: 0.3em; + top: 0.3em; + right: 0.2em; + margin: 0 -1.3em -0.5em 0; +} + +.back_button>div::before, .back_arrow::before { + transform: rotate(-45deg); + top: -0.3em; +} diff --git a/html/base.css b/html/base.css index 2256833..517a5c1 100644 --- a/html/base.css +++ b/html/base.css @@ -1,15 +1,47 @@ /** - * Hachette base styles + * This file is part of Haketilo. + * + * Function: Base styles. * * Copyright (C) 2021 Wojtek Kosior * Copyright (C) 2021 Nicholas Johnson * Redistribution terms are gathered in the `copyright' file. */ +body { + font-family: sans-serif; + background-color: #f0f0f0; + color: #555; + overflow: auto; +} + +.bold, h2 { + font-weight: bold; +} + +h2 { + margin: 8px; + font-size: 120%; +} + +h3 { + padding: 5px; + font-size: 108%; + text-shadow: 0 0 0 #454; +} + +textarea { + font-family: monospace; +} + input[type="checkbox"], input[type="radio"], .hide { display: none; } +.camouflage { + visibility: hidden; +} + .show_next:not(:checked)+* { display: none; } @@ -32,6 +64,10 @@ button, .button { display: inline-block; padding: 6px 12px; margin: 2px 0px; + -moz-user-select: none; + user-select: none; + cursor: pointer; + font: 400 15px sans-serif; } button.slimbutton, .button.slimbutton { @@ -42,3 +78,38 @@ button.slimbutton, .button.slimbutton { button:hover, .button:hover { box-shadow: 0 6px 8px 0 rgba(0,0,0,0.24), 0 17px 50px 0 rgba(0,0,0,0.19); } + +aside { + background: #3f8dc6; + margin: 5px 0; + padding: 0.3em 1em; + border-radius: 3px; + color: #fff; +} + +textarea: { + resize: none; +} + +.has_bottom_line::after, .has_upper_line::before { + content: ""; + display: block; + height: 8px; + background: linear-gradient(transparent, #555); +} + +.has_bottom_line::after { + background: linear-gradient(#555, transparent); +} + +.has_bottom_thin_line { + border-bottom: dashed #4CAF50 1px; +} + +.has_upper_thin_line { + border-top: dashed #4CAF50 1px; +} + +.nowrap { + white-space: nowrap; +} diff --git a/html/default_blocking_policy.html b/html/default_blocking_policy.html new file mode 100644 index 0000000..50c19ca --- /dev/null +++ b/html/default_blocking_policy.html @@ -0,0 +1,18 @@ +<!-- + Copyright (C) 2021 Wojtek Kosior + Redistribution terms are gathered in the `copyright' file. + + This is not a standalone page. This file is meant to be imported into other + HTML code. + --> +<style> + #blocking_policy_div { + line-height: 2em; + } +</style> +<span id="blocking_policy_span"> + Default policy for unmatched pages is to + <span id="current_policy_span" class="bold"></span> + their own scripts. + <button id="toggle_policy_but">Toggle policy</button> +</span> diff --git a/html/default_blocking_policy.js b/html/default_blocking_policy.js new file mode 100644 index 0000000..b6458f3 --- /dev/null +++ b/html/default_blocking_policy.js @@ -0,0 +1,48 @@ +/** + * This file is part of Haketilo. + * + * Function: Logic for the dialog of default policy selection. + * + * Copyright (C) 2021 Wojtek Kosior + * Redistribution terms are gathered in the `copyright' file. + */ + +/* + * IMPORTS_START + * IMPORT by_id + * IMPORT light_storage + * IMPORT observables + * IMPORTS_END + */ + +/* + * Used with `default_blocking_policy.html' to allow user to choose whether to + * block scripts globally or not. + */ + +const blocking_policy_span = by_id("blocking_policy_span"); +const current_policy_span = by_id("current_policy_span"); +const toggle_policy_but = by_id("toggle_policy_but"); + +let policy_observable; + +const update_policy = + allowed => current_policy_span.textContent = allowed ? "allow" : "block"; +const toggle_policy = + () => light_storage.set_var("default_allow", !policy_observable.value); + +async function init_default_policy_dialog() +{ + policy_observable = await light_storage.observe_var("default_allow"); + update_policy(policy_observable.value); + observables.subscribe(policy_observable, update_policy); + + toggle_policy_but.addEventListener("click", toggle_policy); + blocking_policy_span.classList.remove("hide"); +} + +/* + * EXPORTS_START + * EXPORT init_default_policy_dialog + * EXPORTS_END + */ diff --git a/html/display-panel.html b/html/display-panel.html index 1b9c77b..ee9e767 100644 --- a/html/display-panel.html +++ b/html/display-panel.html @@ -1,125 +1,347 @@ <!doctype html> <!-- + This file is part of Haketilo. + + Function: Extension's popup page. + Copyright (C) 2021 Wojtek Kosior Redistribution terms are gathered in the `copyright' file. --> <html> <head> <meta charset="utf-8"/> - <title>Hachette - page settings</title> + <title>Haketilo - page settings</title> + <link type="text/css" rel="stylesheet" href="reset.css" /> <link type="text/css" rel="stylesheet" href="base.css" /> + <link type="text/css" rel="stylesheet" href="back_button.css" /> + <link type="text/css" rel="stylesheet" href="table.css" /> + <link type="text/css" rel="stylesheet" href="MOZILLA_scrollbar_fix.css" /> <style> body { - width: 300px; - height: 300px; + width: max-content; + width: -moz-fit-content; } - ul { - padding-inline-start: 15px; + .top>h2 { + padding-left: calc(0.8*3.2em - 8px); } - .bold { - font-weight: bold; + .top { + line-height: calc(0.8*3.6em - 16px); } - .unroll_chbx:not(:checked)+*+label span.triangle:first-child+span.triangle, - .unroll_chbx:checked+*+label span.triangle:first-child, - .unroll_chbx:not(:checked)+*, - .unroll_chbx:not(:checked)+*+label+* { + #main_view>.top>h2 { + padding-left: 0; + max-width: 550px + } + + .unroll_chbx:not(:checked)+div>:not(:first-child) { display: none; } + .unroll_triangle { + height: 1em; + width: 1em; + display: inline-block; + } + + .unroll_triangle::after { + content: ""; + width: 0.6em; + height: 0.6em; + background: linear-gradient(-45deg, currentColor 50%, transparent 50%); + display: block; + position: relative; + transform: rotate(-45deg); + top: 0.3em; + } + + .unroll_chbx:checked+div>:first-child .unroll_triangle::after { + transform: rotate(45deg); + left: 0.2em; + top: 0.2em; + } + + .unroll_chbx:checked+div>:first-child .unroll_block { + display: block; + } + + .unroll_chbx:checked+div>:first-child { + line-height: 1.4em; + } + + .l2_ul { + border-left: solid #454 5px; + } + + .l1_li { + margin-top: 0.3em; + margin-bottom: 0.3em; + } + + .l1_li>div { + padding: 0.3em 0.3em 0.3em 0; + } + + .l2_li { + padding: 0.3em; + } + + #container_for_injected>*:nth-child(odd), + .l2_li:nth-child(odd) { + background-color: #e5e5e5; + } + #container_for_injected>#none_injected:not(:last-child) { display: none; } + + #page_url_heading>span { + display: inline-block; + } + + .back_button { + position: fixed; + z-index: 1; + top: 0; + left: 0; + /* The following scales the entire button. */ + font-size: 80%; + } + + #show_main_view_radio:checked~.back_button { + margin-left: -3.2em; + } + + #show_main_view_radio:not(:checked)~.back_button { + transition: all 0.2s ease-out; + } + + pre { + font-family: monospace; + background-color: white; + padding: 1px 5px; + } + + .matched_pattern { + font-weight: bold; + } + + tr.matched_pattern~tr { + color: #777; + font-size: 90%; + } + + .padding_inline { + padding-left: 5px; + padding-right: 5px; + } + + .padding_top { + padding-top: 5px; + } + + .header { + padding-bottom: 0.3em; + margin-bottom: 0.5em; + text-align: center; + } + + .middle { + margin-top: 0.5em; + margin-bottom: 0.5em; + } + + .footer { + padding-top: 0.3em; + margin-top: 0.5em; + text-align: center; + } + + .active_setting_table { + margin-bottom: 0.5em; + } + + .active_setting_table td { + padding: 5px; + vertical-align: middle; + } </style> </head> <body> - <!-- The invisible div below is for elements that will be cloned. --> - <div class="hide"> - <li id="pattern_li_template"> - <span></span> - <button>View in settings</button> - </li> - <li id="query_match_li_template" class="queried_pattern_match" data-template="li"> + <template> + <tr id="pattern_entry" class="nowrap" data-template="entry"> + <td data-template="name"></td> + <td> + <div class="button" data-template="button">Add setting</div> + </td> + </tr> + + <li id="query_match_li" class="l2_li" data-template="li"> <div> <span>pattern:</span> <span class="bold" data-template="pattern"></span> - <button data-template="btn">Install</button> + <label class="button slimbutton" for="show_install_view_radio" data-template="btn"> + Install + </label> </div> - <div id="unrollable_component_template" data-template="unroll_container"> - <span data-template="component_label">payload:</span> + <div id="unrollable_component" data-template="unroll_container"> <input type="checkbox" class="unroll_chbx" data-template="chbx"></input> - <br data-template="br"/> - <label class="bold" data-template="lbl"> - <span data-template="triangle"> - <span class="triangle">⏵</span> - <span class="triangle">⏷</span> + <div> + <span>payload: + <label class="bold unroll_block" data-template="lbl"> + <div data-template="triangle" class="unroll_triangle"></div> + <span data-template="payload"></span> + </label> </span> - <span data-template="component"></span> + <div data-template="unroll"></div> + </div> + </div> + </li> + + <div id="injected_script" data-template="div"> + <input type="checkbox" class="unroll_chbx" data-template="chbx"></input> + <div> + <label data-template="lbl"> + <h3><div class="unroll_triangle"></div> script</h3> </label> - <div data-template="unroll"></div> + <pre class="has_bottom_thin_line has_upper_thin_line" data-template="script_contents"></pre> + </div> + </div> + + <div id="multi_repos_query_result" data-template="div"> + Results for <span class="bold" data-template="url_span"></span> + <ul class="l1_ul" data-template="ul"></ul> + </div> + + <li id="single_repo_query_result" class="l1_li" data-template="li"> + <div> + From <span class="bold" data-template="repo_url"></span> </div> </li> - </div> - <input id="show_install_view_chbx" type="checkbox" class="show_hide_next2"></input> + <ul id="result_patterns_list" class="l2_ul" data-template="ul"> + </ul> + </template> + + <input id="show_install_view_radio" type="radio" class="show_next" name="current_view"></input> <div id="install_view"> - <IMPORT html/import_frame.html /> - <!-- - <div id="install_status"></div> - <label for="show_install_chbx" class="bold">Cancel install</label> - <button id="commit_install_but">Commit install</button> - --> + <div class="top has_bottom_line"><h2> Site modifiers install </h2></div> + <div class="padding_inline"> + <IMPORT html/import_frame.html /> + </div> </div> - <div id="main_view"> - <h2 id="page_url_heading"></h2> - - <input id="show_privileged_notice_chbx" type="checkbox" class="show_next"></input> - <h3>Privileged page</h3> - - <input id="show_page_state_chbx" type="checkbox" class="show_next"></input> - <div> - <input id="possible_patterns_chbx" type="checkbox" class="unroll_chbx"></input> - <span></span> - <label for="possible_patterns_chbx"> - <h3> - <span class="triangle">⏵</span> - <span class="triangle">⏷</span> - Possible patterns - </h3> - </label> - <ul id="possible_patterns"></ul> - - <input id="connected_chbx" type="checkbox" class="show_hide_next2"></input> + + <input id="show_injected_view_radio" type="radio" class="show_next" name="current_view"></input> + <div id="injected_view"> + <div class="top has_bottom_line"><h2>Injected scripts</h2></div> + <div id="container_for_injected"> + <span id="none_injected">None</span> + </div> + </div> + + <input id="show_patterns_view_radio" type="radio" class="show_next" name="current_view"></input> + <div> + <div class="top has_bottom_line"><h2>Possible patterns for this page</h2></div> + <div class="padding_inline"> + <aside> + Patterns higher are more specific and override the ones below. + </aside> + </div> + <div class="table_wrapper firefox_scrollbars_hacky_fix"> <div> - Matched pattern: <span id="pattern" class="bold">...</span> - <button id="view_pattern" class="hide"> - View in settings - </button> - <br/> - Blocked: <span id="blocked" class="bold">...</span> - <br/> - Payload: <span id="payload" class="bold">...</span> - <button id="view_payload" class="hide"> - View in settings - </button> - <h3>Injected</h3> - <div id="container_for_injected"> - <span id="none_injected">None</span> + <table> + <tbody id="possible_patterns"> + </tbody> + </table> + </div> + </div> + <div class="padding_inline padding_top has_upper_thin_line firefox_scrollbars_hacky_fix has_inline_content"> + <span class="nowrap"> + <IMPORT html/default_blocking_policy.html /> + </span> + </div> + </div> + + <input id="show_queried_view_radio" type="radio" class="show_next" name="current_view"></input> + <div> + <div class="top has_bottom_line"><h2>Queried from repositories</h2></div> + <div id="container_for_repo_responses" class="padding_inline"> + </div> + </div> + + <input id="show_main_view_radio" type="radio" class="show_next" name="current_view" checked></input> + <div id="main_view"> + <div class="top has_bottom_line"><h2 id="page_url_heading"></h2></div> + <h3 id="privileged_notice" class="middle hide">Privileged page</h3> + + <div id="page_state" class="hide"> + <div class="header padding_inline has_bottom_thin_line"> + <label for="show_patterns_view_radio" class="button"> + Edit settings for this page + </label> + </div> + <div class="middle padding_inline"> + <input id="connected_chbx" type="checkbox" class="show_hide_next2"></input> + <div> + <table class="active_setting_table"> + <tbody> + <tr class="nowrap"> + <td>Matched pattern:</td> + <td id="pattern" class="bold">...</td> + <td> + <button id="view_pattern" class="hide"> + View in settings + </button> + </td> + </tr> + <tr class="nowrap"> + <td>Scripts blocked:</td> + <td id="blocked" class="bold">...</td> + <td></td> + </tr> + <tr class="nowrap"> + <td>Injected payload:</td> + <td id="payload" class="bold">...</td> + <td id="payload_buttons" class="hide"> + <button id="view_payload"> View in settings </button> + <br/> + <label id="view_injected" class="button" for="show_injected_view_radio"> + View injected scripts + </label> + </td> + </tr> + <tr> + <td id="content_type" colspan="3" class="hide"> + This is a non-HTML page. Chosen payload will not be injected. + </td> + </tr> + </tbody> + </table> + <label id="query_pattern" for="show_queried_view_radio" class="button"> + Install scripts for this page + </label> </div> - <input id="query_started_chbx" type="checkbox" class="show_hide_next2"></input> - <div id="container_for_repo_responses"> - <h3>Queried from repositories</h3> + <div> + <h3> + Connecting to content script..<span id="loading_point">.</span> + </h3> + <aside id="reload_notice"> + Try reloading the page. + </aside> </div> - <button id="query_pattern"> - Search for matching patterns - </button> </div> - <h3>Trying to connect..<input id="loading_chbx" type="checkbox" class="show_next"></input><span>.</span></h3> </div> - <button id="settings_but" type="button" style="margin-top: 20px;">Settings</button> - </div>_POPUPSCRIPTS_ + <div class="footer padding_inline has_upper_thin_line"> + <button id="settings_but" type="button"> + Open Haketilo settings + </button> + </div> + </div> + + <div class="has_upper_line"></div> + + <label for="show_main_view_radio" class="back_button"><div></div></label>_POPUPSCRIPTS_ </body> </html> diff --git a/html/display-panel.js b/html/display-panel.js index b4d9abb..4fe0173 100644 --- a/html/display-panel.js +++ b/html/display-panel.js @@ -1,5 +1,7 @@ /** - * Hachette display panel logic + * This file is part of Haketilo. + * + * Function: Popup logic. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. @@ -14,14 +16,14 @@ *** temporarily, before all storage access gets reworked. * IMPORT get_remote_storage * IMPORT get_import_frame + * IMPORT init_default_policy_dialog * IMPORT query_all * IMPORT CONNECTION_TYPE - * IMPORT url_item * IMPORT is_privileged_url * IMPORT TYPE_PREFIX * IMPORT nice_name * IMPORT open_in_settings - * IMPORT for_each_possible_pattern + * IMPORT each_url_pattern * IMPORT by_id * IMPORT clone_template * IMPORTS_END @@ -30,6 +32,16 @@ let storage; let tab_url; +/* Force popup <html>'s reflow on stupid Firefox. */ +if (is_mozilla) { + const reflow_forcer = + () => document.documentElement.style.width = "-moz-fit-content"; + for (const radio of document.querySelectorAll('[name="current_view"]')) + radio.addEventListener("change", reflow_forcer); +} + +const show_queried_view_radio = by_id("show_queried_view_radio"); + const tab_query = {currentWindow: true, active: true}; async function get_current_tab() @@ -48,8 +60,21 @@ async function get_current_tab() } const page_url_heading = by_id("page_url_heading"); -const show_privileged_notice_chbx = by_id("show_privileged_notice_chbx"); -const show_page_state_chbx = by_id("show_page_state_chbx"); +const privileged_notice = by_id("privileged_notice"); +const page_state = by_id("page_state"); + +/* Helper functions to convert string into a list of one-letter <span>'s. */ +function char_to_span(char, doc) +{ + const span = document.createElement("span"); + span.textContent = char; + return span; +} + +function to_spans(string, doc=document) +{ + return string.split("").map(c => char_to_span(c, doc)); +} async function show_page_activity_info() { @@ -60,80 +85,60 @@ async function show_page_activity_info() return; } - tab_url = url_item(tab.url); - page_url_heading.textContent = tab_url; + tab_url = /^([^?#]*)/.exec(tab.url)[1]; + to_spans(tab_url).forEach(s => page_url_heading.append(s)); if (is_privileged_url(tab_url)) { - show_privileged_notice_chbx.checked = true; + privileged_notice.classList.remove("hide"); return; } populate_possible_patterns_list(tab_url); - show_page_state_chbx.checked = true; + page_state.classList.remove("hide"); try_to_connect(tab.id); } -const possible_patterns_ul = by_id("possible_patterns"); -const pattern_li_template = by_id("pattern_li_template"); -pattern_li_template.removeAttribute("id"); +const possible_patterns_list = by_id("possible_patterns"); const known_patterns = new Map(); function add_pattern_to_list(pattern) { - const li = pattern_li_template.cloneNode(true); - li.id = `pattern_li_${known_patterns.size}`; - known_patterns.set(pattern, li.id); + const template = clone_template("pattern_entry"); + template.name.textContent = pattern; - const span = li.firstElementChild; - span.textContent = pattern; - - const button = span.nextElementSibling; const settings_opener = () => open_in_settings(TYPE_PREFIX.PAGE, pattern); - button.addEventListener("click", settings_opener); + template.button.addEventListener("click", settings_opener); - possible_patterns_ul.appendChild(li) + known_patterns.set(pattern, template); + possible_patterns_list.append(template.entry); - return li.id; + return template; } -function ensure_pattern_exists(pattern) +function style_possible_pattern_entry(pattern, exists_in_settings) { - let id = known_patterns.get(pattern); - /* - * As long as pattern computation works well, we should never get into this - * conditional block. This is just a safety measure. To be removed as part - * of a bigger rework when we start taking iframes into account. - */ - if (id === undefined) { - console.log(`unknown pattern: ${pattern}`); - id = add_pattern_to_list(pattern); - } + const [text, class_action] = exists_in_settings ? + ["Edit", "add"] : ["Add", "remove"]; + const entry_object = known_patterns.get(pattern); - return id; -} - -function set_pattern_li_button_text(li_id, text) -{ - by_id(li_id).firstElementChild.nextElementSibling.textContent = text; + if (entry_object) { + entry_object.button.textContent = `${text} setting`; + entry_object.entry.classList[class_action]("matched_pattern"); + } } function handle_page_change(change) { - const li_id = ensure_pattern_exists(change.item); - if (change.old_val === undefined) - set_pattern_li_button_text(li_id, "Edit in settings"); - if (change.new_val === undefined) - set_pattern_li_button_text(li_id, "Add setting"); + style_possible_pattern_entry(change.item, change.new_val !== undefined); } function populate_possible_patterns_list(url) { - for_each_possible_pattern(url, add_pattern_to_list); + for (const pattern of each_url_pattern(url)) + add_pattern_to_list(pattern); - for (const [pattern, settings] of query_all(storage, url)) { - set_pattern_li_button_text(ensure_pattern_exists(pattern), - "Edit in settings"); - } + for (const [pattern, settings] of query_all(storage, url)) + style_possible_pattern_entry(pattern, true); storage.add_change_listener(handle_page_change, [TYPE_PREFIX.PAGE]); } @@ -159,17 +164,16 @@ function try_to_connect(tab_id) setTimeout(() => monitor_connecting(tab_id), 1000); } -const query_started_chbx = by_id("query_started_chbx"); - -function start_querying_repos(port) +function start_querying_repos() { + query_pattern_but.removeEventListener("click", start_querying_repos); const repo_urls = storage.get_all_names(TYPE_PREFIX.REPO); if (content_script_port) content_script_port.postMessage([TYPE_PREFIX.URL, tab_url, repo_urls]); - query_started_chbx.checked = true; } -const loading_chbx = by_id("loading_chbx"); +const loading_point = by_id("loading_point"); +const reload_notice = by_id("reload_notice"); function handle_disconnect(tab_id, button_cb) { @@ -183,7 +187,9 @@ function handle_disconnect(tab_id, button_cb) if (connected_chbx.checked) return; - loading_chbx.checked = !loading_chbx.checked; + loading_point.classList.toggle("camouflage"); + reload_notice.classList.remove("hide"); + setTimeout(() => try_to_connect(tab_id), 1000); } @@ -197,7 +203,8 @@ function monitor_connecting(tab_id) else return; - loading_chbx.checked = !loading_chbx.checked; + loading_point.classList.toggle("camouflage"); + reload_notice.classList.remove("hide"); try_to_connect(tab_id); } @@ -205,11 +212,16 @@ const pattern_span = by_id("pattern"); const view_pattern_but = by_id("view_pattern"); const blocked_span = by_id("blocked"); const payload_span = by_id("payload"); +const payload_buttons_div = by_id("payload_buttons"); const view_payload_but = by_id("view_payload"); +const view_injected_but = by_id("view_injected"); const container_for_injected = by_id("container_for_injected"); +const content_type_cell = by_id("content_type"); const queried_items = new Map(); +let max_injected_script_id = 0; + function handle_activity_report(message) { connected_chbx.checked = true; @@ -217,43 +229,43 @@ function handle_activity_report(message) const [type, data] = message; if (type === "settings") { - let [pattern, settings] = data; + const settings = data; - settings = settings || {}; blocked_span.textContent = settings.allow ? "no" : "yes"; - if (pattern) { + if (settings.pattern) { pattern_span.textContent = pattern; const settings_opener = - () => open_in_settings(TYPE_PREFIX.PAGE, pattern); + () => open_in_settings(TYPE_PREFIX.PAGE, settings.pattern); view_pattern_but.classList.remove("hide"); view_pattern_but.addEventListener("click", settings_opener); } else { pattern_span.textContent = "none"; + blocked_span.textContent = blocked_span.textContent + " (default)"; } - const components = settings.components; - if (components) { - payload_span.textContent = nice_name(...components); - const settings_opener = () => open_in_settings(...components); - view_payload_but.classList.remove("hide"); + if (settings.payload) { + payload_span.textContent = nice_name(...settings.payload); + payload_buttons_div.classList.remove("hide"); + const settings_opener = () => open_in_settings(...settings.payload); view_payload_but.addEventListener("click", settings_opener); } else { payload_span.textContent = "none"; } } if (type === "script") { - const h4 = document.createElement("h4"); - const pre = document.createElement("pre"); - h4.textContent = "script"; - pre.textContent = data; - - container_for_injected.appendChild(h4); - container_for_injected.appendChild(pre); + const template = clone_template("injected_script"); + const chbx_id = `injected_script_${max_injected_script_id++}`; + template.chbx.id = chbx_id; + template.lbl.setAttribute("for", chbx_id); + template.script_contents.textContent = data; + container_for_injected.appendChild(template.div); + } + if (type === "is_html") { + if (!data) + content_type_cell.classList.remove("hide"); } if (type === "repo_query_action") { - query_started_chbx.checked = true; - const key = data.prefix + data.item; const results = queried_items.get(key) || {}; Object.assign(results, data.results); @@ -273,35 +285,26 @@ const results_lists = new Map(); function create_results_list(url) { - const list_div = document.createElement("div"); - const list_head = document.createElement("h4"); - const list = document.createElement("ul"); + const cloned_template = clone_template("multi_repos_query_result"); + cloned_template.url_span.textContent = url; + container_for_repo_responses.appendChild(cloned_template.div); - list_head.textContent = url; - list_div.appendChild(list_head); - list_div.appendChild(list); - container_for_repo_responses.appendChild(list_div); + cloned_template.by_repo = new Map(); + results_lists.set(url, cloned_template); - const list_object = {list, by_repo: new Map()}; - - results_lists.set(url, list_object); - - return list_object; + return cloned_template; } function create_result_item(list_object, repo_url, result) { - const result_li = document.createElement("li"); - const repo_url_span = document.createElement("span"); - const result_item = {result_li, appended: null}; + const cloned_template = clone_template("single_repo_query_result"); + cloned_template.repo_url.textContent = repo_url; + cloned_template.appended = null; - repo_url_span.textContent = repo_url; - result_li.appendChild(repo_url_span); + list_object.ul.appendChild(cloned_template.li); + list_object.by_repo.set(repo_url, cloned_template); - list_object.list.appendChild(result_li); - list_object.by_repo.set(repo_url, result_item); - - return result_item; + return cloned_template; } function set_appended(result_item, element) @@ -309,7 +312,7 @@ function set_appended(result_item, element) if (result_item.appended) result_item.appended.remove(); result_item.appended = element; - result_item.result_li.appendChild(element); + result_item.li.appendChild(element); } function show_message(result_item, text) @@ -332,11 +335,9 @@ function unroll_chbx_first_checked(entry_object) entry_object.chbx.removeEventListener("change", entry_object.unroll_cb); delete entry_object.unroll_cb; - entry_object.unroll.textContent = "preview not implemented..."; + entry_object.unroll.innerHTML = "preview not implemented...<br />(consider contributing)"; } -const show_install_chbx = by_id("show_install_view_chbx"); - let import_frame; let install_target = null; @@ -462,7 +463,6 @@ function record_fetched_install_dep(prefix, item, repo_url, result) function install_clicked(entry_object) { - show_install_chbx.checked = true; import_frame.show_loading(); install_target = { @@ -482,25 +482,29 @@ var max_query_result_id = 0; function show_query_successful_result(result_item, repo_url, result) { - const ul = document.createElement("ul"); + if (result.length === 0) { + show_message(result_item, "No results :("); + return; + } - set_appended(result_item, ul); + const cloned_ul_template = clone_template("result_patterns_list"); + set_appended(result_item, cloned_ul_template.ul); for (const match of result) { - const entry_object = clone_template("query_match_li_template"); + const entry_object = clone_template("query_match_li"); entry_object.pattern.textContent = match.pattern; - ul.appendChild(entry_object.li); + cloned_ul_template.ul.appendChild(entry_object.li); if (!match.payload) { entry_object.payload.textContent = "(none)"; - for (const key of ["chbx", "br", "triangle", "unroll"]) + for (const key of ["chbx", "triangle", "unroll"]) entry_object[key].remove(); continue; } - entry_object.component.textContent = nice_name(...match.payload); + entry_object.payload.textContent = nice_name(...match.payload); const install_cb = () => install_clicked(entry_object); entry_object.btn.addEventListener("click", install_cb); @@ -541,9 +545,11 @@ by_id("settings_but") async function main() { + init_default_policy_dialog(); + storage = await get_remote_storage(); import_frame = await get_import_frame(); - import_frame.onclose = () => show_install_chbx.checked = false; + import_frame.onclose = () => show_queried_view_radio.checked = true; show_page_activity_info(); } diff --git a/html/import_frame.html b/html/import_frame.html index c86c3de..754b289 100644 --- a/html/import_frame.html +++ b/html/import_frame.html @@ -1,13 +1,28 @@ -<div style="display: none;"> - <li id="import_li_template"> - <span></span> - <input type="checkbox" style="display: inline;" checked></input> - <span></span> - </li> -</div> -<h2> Settings import </h2> +<!-- + Copyright (C) 2021 Wojtek Kosior + Redistribution terms are gathered in the `copyright' file. + + This is not a standalone page. This file is meant to be imported into other + HTML code. + --> +<style> + .padding_right { + padding-right: 0.3em; + } +</style> +<template> + <tr id="import_entry" class="nowrap" data-template="entry"> + <td> + <input type="checkbox" style="display: inline;" checked data-template="chbx"></input> + <span data-template="name_span"></span> + </td> + <td class="bold padding_right" data-template="warning"></td> + </tr> +</template> + <input id="import_loading_radio" type="radio" name="import_window_content" class="show_next"></input> <span> Loading... </span> + <input id="import_failed_radio" type="radio" name="import_window_content" class="show_next"></input> <div> <span id="import_errormsg"></span> @@ -15,13 +30,23 @@ <pre id="import_errordetail"></pre> <button id="import_failok_but"> OK </button> </div> + <input id="import_selection_radio" type="radio" name="import_window_content" class="show_next"></input> <div> <button id="check_all_import_but"> Check all </button> <button id="uncheck_all_import_but"> Uncheck all </button> <button id="uncheck_colliding_import_but"> Uncheck existing </button> - <ul id="import_ul"> - </ul> + <aside id="existing_settings_note"> + Settings that would owerwrite existing ones are marked "!". + </aside> + <div id="import_table_wrapper" class="table_wrapper"> + <div> + <table> + <tbody id="import_list"> + </tbody> + </table> + </div> + </div> <button id="commit_import_but"> OK </button> <button id="cancel_import_but"> Cancel </button> </div> diff --git a/html/import_frame.js b/html/import_frame.js index 4075433..ae6fab4 100644 --- a/html/import_frame.js +++ b/html/import_frame.js @@ -1,5 +1,7 @@ /** - * Hachette HTML import frame script + * This file is part of Haketilo. + * + * Function: Logic for the settings import frame. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. @@ -9,6 +11,7 @@ * IMPORTS_START * IMPORT get_remote_storage * IMPORT by_id + * IMPORT clone_template * IMPORT nice_name * IMPORT make_once * IMPORTS_END @@ -16,48 +19,38 @@ let storage; -const import_li_template = by_id("import_li_template"); -import_li_template.removeAttribute("id"); - -function import_li_id(prefix, item) -{ - return `ili_${prefix}_${item}`; -} - -let import_ul = by_id("import_ul"); +let import_list = by_id("import_list"); let import_chbxs_colliding = undefined; +let entry_objects = undefined; let settings_import_map = undefined; -function add_import_li(prefix, name) +function add_import_entry(prefix, name) { - let li = import_li_template.cloneNode(true); - let name_span = li.firstElementChild; - let chbx = name_span.nextElementSibling; - let warning_span = chbx.nextElementSibling; + const cloned_template = clone_template("import_entry"); + Object.assign(cloned_template, {prefix, name}); - li.setAttribute("data-prefix", prefix); - li.setAttribute("data-name", name); - li.id = import_li_id(prefix, name); - name_span.textContent = nice_name(prefix, name); + cloned_template.name_span.textContent = nice_name(prefix, name); if (storage.get(prefix, name) !== undefined) { - import_chbxs_colliding.push(chbx); - warning_span.textContent = "(will overwrite existing setting!)"; + import_chbxs_colliding.push(cloned_template.chbx); + cloned_template.warning.textContent = "!"; } - import_ul.appendChild(li); + import_list.appendChild(cloned_template.entry); + + return cloned_template; } function check_all_imports() { - for (let li of import_ul.children) - li.firstElementChild.nextElementSibling.checked = true; + for (const entry_object of entry_objects) + entry_object.chbx.checked = true; } function uncheck_all_imports() { - for (let li of import_ul.children) - li.firstElementChild.nextElementSibling.checked = false; + for (const entry_object of entry_objects) + entry_object.chbx.checked = false; } function uncheck_colliding_imports() @@ -68,17 +61,13 @@ function uncheck_colliding_imports() function commit_import() { - for (let li of import_ul.children) { - let chbx = li.firstElementChild.nextElementSibling; - - if (!chbx.checked) + for (const entry_object of entry_objects) { + if (!entry_object.chbx.checked) continue; - let prefix = li.getAttribute("data-prefix"); - let name = li.getAttribute("data-name"); - let key = prefix + name; - let value = settings_import_map.get(key); - storage.set(prefix, name, value); + const key = entry_object.prefix + entry_object.name; + const value = settings_import_map.get(key); + storage.set(entry_object.prefix, entry_object.name, value); } deactivate(); @@ -105,38 +94,48 @@ function show_error(errormsg, errordetail) } const import_selection_radio = by_id("import_selection_radio"); +const existing_settings_note = by_id("existing_settings_note"); function show_selection(settings) { import_selection_radio.checked = true; - let old_children = import_ul.children; + let old_children = import_list.children; while (old_children[0] !== undefined) - import_ul.removeChild(old_children[0]); + import_list.removeChild(old_children[0]); import_chbxs_colliding = []; + entry_objects = []; settings_import_map = new Map(); for (let setting of settings) { let [key, value] = Object.entries(setting)[0]; let prefix = key[0]; let name = key.substring(1); - add_import_li(prefix, name); + entry_objects.push(add_import_entry(prefix, name)); settings_import_map.set(key, value); } + + const op = import_chbxs_colliding.length > 0 ? "remove" : "add"; + existing_settings_note.classList[op]("hide"); } function deactivate() { /* Let GC free some memory */ import_chbxs_colliding = undefined; + entry_objects = undefined; settings_import_map = undefined; if (exports.onclose) exports.onclose(); } -const exports = {show_loading, show_error, show_selection, deactivate}; +const wrapper = by_id("import_table_wrapper"); +const style_table = (...cls) => cls.forEach(c => wrapper.classList.add(c)); + +const exports = + {show_loading, show_error, show_selection, deactivate, style_table}; async function init() { diff --git a/html/options.html b/html/options.html index 2246f9a..2e8317c 100644 --- a/html/options.html +++ b/html/options.html @@ -1,32 +1,22 @@ <!doctype html> <!-- + This file is part of Haketilo. + + Function: Extension's settings page. + Copyright (C) 2021 Wojtek Kosior - Copyright (C) 2021 Nicholas Johnson Redistribution terms are gathered in the `copyright' file. --> <html> <head> <meta charset="utf-8"/> - <title>Hachette options</title> + <title>Haketilo options</title> + <link type="text/css" rel="stylesheet" href="reset.css" /> <link type="text/css" rel="stylesheet" href="base.css" /> + <link type="text/css" rel="stylesheet" href="table.css" /> <style> - /* pages list */ - #page_components_ul { - max-height: 80vh; - overflow-y: auto; - } - #page_components_ul li.dragover_li { - border-top: 2px solid blue; - } - #page_components_ul li { - border-top: 2px solid white; - } - li[draggable=true] * { - pointer-events: none; - } - li[draggable=true] label, - li[draggable=true] button { - pointer-events: auto; + body { + width: 100%; } /* tabbed view */ @@ -37,20 +27,38 @@ display: none; } - #show_repos:checked ~ #repos_lbl, - #show_pages:checked ~ #pages_lbl, - #show_bags:checked ~ #bags_lbl, - #show_scripts:checked ~ #scripts_lbl { + #show_repos:checked ~ div #repos_lbl, + #show_pages:checked ~ div #pages_lbl, + #show_bags:checked ~ div #bags_lbl, + #show_scripts:checked ~ div #scripts_lbl { background: #4CAF50; color: white; } - body > div { - border-top: 6px solid #4CAF50; + #tab_heads>* { + font-size: 130%; + padding: 10px; + display: inline-block; + cursor: pointer; } - .tab_head { - display: inline-block; + #tab_heads { + -moz-user-select: none; + user-select: none; + } + + #import_but { + font: unset; + font-size: 130%; + float: right; + margin: 0; + border-radius: 0; + } + + div.tab { + min-width: 50vw; + width: fit-content; + padding-left: 6px; } /* popup window with list of selectable components */ @@ -60,7 +68,7 @@ height: 100vh; left: 0; top: 0; - background-color: rgba(0,0,0,0.4); + background-color: rgba(60,60,60,0.4); z-index: 1; overflow: auto; vertical-align: center; @@ -68,135 +76,284 @@ } .popup_frame { - background-color: white; - width: 50vw; + background-color: #f0f0f0; + margin: 5vh auto; + padding: 15px; + border: solid #333 4px; + border-radius: 15px; + width: -moz-fit-content; + width: fit-content; + } + + .work_li .table_wrapper::before { + background: linear-gradient(#e0f0f0, #555); + } + + .work_li .table_wrapper::after { + background: linear-gradient(#555, #e0f0f0); + } + + .table_wrapper.always_scrollbar>* { + border-left: solid #454 8px; + max-height: 80vh; + overflow-y: scroll; + } + + .table_wrapper .table_wrapper.always_scrollbar>*, + .popup_frame .table_wrapper.always_scrollbar>* { + max-height: 60vh; + } + + .popup_frame .table_wrapper table { + min-width: 30vw; + } + + .popup_frame .table_wrapper { + margin: 0 auto; + } + + td:first-child { + max-width: 70vw; + overflow: hidden; + } + + tr.work_li>td:first-child { + padding-right: 0; + max-width: unset; } - input[type="radio"]:not(:checked)+.import_window_content { + tr.work_li>td>div { + background-color: #e0f0f0; + border-left: solid #454 5px; + border-right: solid #454 2px; + padding: 5px 10px; + } + + .form_grid { + display: grid; + grid-template-columns: auto auto; + } + + .form_grid>label { + grid-column: 1 / span 1; + margin-right: 10px; + } + + .form_grid label { + line-height: 34px; /* center text vertically */ + } + + .form_grid>input, .form_grid>span { + grid-column: 2 / span 1; + } + + .form_grid>label[for="script_contents_field"], + .form_grid>* { + grid-column: 1 / span 2; + } + + .form_grid>textarea { + min-width: 70vw; + resize: none; + } + + .form_disabled>* { + opacity: 0.5; + pointer-events: none; + } + + .form_disabled_msg { display: none; + font-style: italic; + } + + .form_disabled .form_disabled_msg { + opacity: initial; + pointer-events: initial; + display: initial; } </style> </head> <body> - <!-- The invisible div below is for elements that will be cloned. --> - <div class="hide"> - <li id="item_li_template"> - <span></span> - <button> Edit </button> - <button> Remove </button> - <button> Export </button> - </li> - <li id="bag_component_li_template"> - <span></span> - <button> Remove </button> - </li> - <li id="chbx_component_li_template"> - <input type="checkbox" style="display: inline;"></input> - <span></span> - </li> - <li id="radio_component_li_template"> - <input type="radio" style="display: inline;" name="page_components"></input> - <span></span> - </li> - </div> + <template> + <tr id="item_li" class="nowrap"> + <td></td> + <td><div class="button"> Edit </div></td> + <td><div class="button"> Remove </div></td> + <td><div class="button"> Export </div></td> + </tr> + <tr id="bag_component_li" class="nowrap"> + <td></td> + <td><div class="button"> Remove </div></td> + </tr> + <tr id="chbx_component_li" class="nowrap"> + <td> + <input type="checkbox" style="display: inline;"></input> + <span></span> + </td> + </tr> + <tr id="radio_component_li" class="nowrap"> + <td> + <input type="radio" style="display: inline;" name="page_components"></input> + <span></span> + </td> + </tr> + </template> <!-- Mind the show_*s ids below - their format is assumed in js code --> <input type="radio" name="tabs" id="show_repos"></input> <input type="radio" name="tabs" id="show_pages" checked></input> <input type="radio" name="tabs" id="show_bags"></input> <input type="radio" name="tabs" id="show_scripts"></input> - <label for="show_repos" id="repos_lbl" - class="tab_head"> Repos </label> - <label for="show_pages" id="pages_lbl" - class="tab_head"> Pages </label> - <label for="show_bags" id="bags_lbl" - class="tab_head"> Bags </label> - <label for="show_scripts" id="scripts_lbl" - class="tab_head"> Scripts </label> - - <div id="repos"> - <ul id="repos_ul"> - <li id="work_repo_li" class="hide"> - <label for="repo_url_field">URL: </label> - <input id="repo_url_field"></input> - <br/> - <button id="save_repo_but" type="button"> Save </button> - <button id="discard_repo_but" type="button"> Cancel </button> - </li> - </ul> + <div id="tab_heads" class="has_bottom_line"> + <label for="show_repos" id="repos_lbl"> Repos </label> + <label for="show_pages" id="pages_lbl"> Pages </label> + <label for="show_bags" id="bags_lbl"> Bags </label> + <label for="show_scripts" id="scripts_lbl"> Scripts </label> + <button id="import_but" style="margin-left: 40px;"> Import </button> + </div> + <div id="repos" class="tab"> + <div class="table_wrapper tight_table has_bottom_line has_upper_line"> + <div> + <table> + <tbody id="repos_ul"> + <tr id="work_repo_li" class="hide work_li"> + <td colspan="4"> + <div class="form_grid"> + <label for="repo_url_field">URL: </label> + <input id="repo_url_field"></input> + <div> + <button id="save_repo_but" type="button"> Save </button> + <button id="discard_repo_but" type="button"> Cancel </button> + </div> + </div> + </td> + </tr> + </tbody> + </table> + </div> + </div> <button id="add_repo_but" type="button"> Add repository </button> </div> - - <div id="pages"> - <ul id="pages_ul"> - <li id="work_page_li" class="hide"> - <label for="page_url_field">URL: </label> - <input id="page_url_field"></input> - <br/> - <label>Payload: </label> - <span id="page_payload"></span> - <button id="select_page_components_but"> - Choose payload - </button> - <br/> - <input id="page_allow_chbx" type="checkbox" style="display: inline;"></input> - <label for="page_allow_chbx">Allow native scripts</label> - <br/> - <button id="save_page_but" type="button"> Save </button> - <button id="discard_page_but" type="button"> Cancel </button> - </li> - </ul> + <div id="pages" class="tab"> + <div class="table_wrapper tight_table has_bottom_line has_upper_line"> + <div> + <table> + <tbody id="pages_ul"> + <tr id="work_page_li" class="hide work_li"> + <td colspan="4"> + <div class="form_grid"> + <label for="page_url_field">URL: </label> + <input id="page_url_field"></input> + <label>Payload: </label> + <span class="nowrap"> + <span id="page_payload"></span> + <button id="select_page_components_but"> + Choose payload + </button> + </span> + <div id="allow_native_scripts_container" class="nowrap"> + <input id="page_allow_chbx" type="checkbox" style="display: inline;"></input> + <label for="page_allow_chbx">Allow native scripts</label> + <span class="form_disabled_msg"> + (only possible when no payload is used) + </span> + </div> + <div> + <button id="save_page_but" type="button"> Save </button> + <button id="discard_page_but" type="button"> Cancel </button> + </div> + </div> + </td> + </tr> + </tbody> + </table> + </div> + </div> <button id="add_page_but" type="button"> Add page </button> + <br/> + <IMPORT html/default_blocking_policy.html /> </div> - - <div id="bags"> - <ul id="bags_ul"> - <li id="work_bag_li" class="hide"> - <label for="bag_name_field"> Name: </label> - <input id="bag_name_field"></input> - <ul id="bag_components_ul"> - <li id="empty_bag_component_li" class="hide"></li> - </ul> - <button id="select_bag_components_but"> - Add scripts - </button> - <br/> - <button id="save_bag_but"> Save </button> - <button id="discard_bag_but"> Cancel </button> - </li> - </ul> + <div id="bags" class="tab"> + <div class="table_wrapper tight_table has_bottom_line has_upper_line"> + <div> + <table> + <tbody id="bags_ul"> + <tr id="work_bag_li" class="hide work_li"> + <td colspan="4"> + <div class="form_grid"> + <label for="bag_name_field"> Name: </label> + <input id="bag_name_field"></input> + <div class="table_wrapper tight_table has_bottom_line has_upper_line"> + <div> + <table> + <tbody id="bag_components_ul"> + <tr id="empty_bag_component_li" class="hide"></tr> + </tbody> + </table> + </div> + </div> + <div> + <button id="select_bag_components_but"> + Add scripts + </button> + </div> + <div> + <button id="save_bag_but"> Save </button> + <button id="discard_bag_but"> Cancel </button> + </div> + </div> + </td> + </tr> + </tbody> + </table> + </div> + </div> <button id="add_bag_but" type="button"> Add bag </button> </div> - - <div id="scripts"> - <ul id="scripts_ul"> - <li id="work_script_li" class="hide"> - <label for="script_name_field"> Name: </label> - <input id="script_name_field"></input> - <br/> - <label for="script_url_field"> URL: </label> - <input id="script_url_field"></input> - <br/> - <label for="script_sha256_field"> sha256: </label> - <input id="script_sha256_field"></input> - <br/> - <label for="script_contents_field"> contents: </label> - <textarea id="script_contents_field" rows="20" cols="80"></textarea> - <br/> - <button id="save_script_but"> Save </button> - <button id="discard_script_but"> Cancel </button> - </li> - </ul> + <div id="scripts" class="tab"> + <div class="table_wrapper tight_table has_bottom_line has_upper_line"> + <div> + <table> + <tbody id="scripts_ul"> + <tr id="work_script_li" class="hide work_li"> + <td colspan="4"> + <div class="form_grid"> + <label for="script_name_field"> Name: </label> + <input id="script_name_field"></input> + <label for="script_url_field"> URL: </label> + <input id="script_url_field"></input> + <label for="script_sha256_field"> SHA256: </label> + <input id="script_sha256_field"></input> + <aside> + Note: URL and SHA256 are ignored if script text is provided. + </aside> + <label for="script_contents_field"> contents: </label> + <textarea id="script_contents_field" rows="20" cols="80"></textarea> + <div> + <button id="save_script_but"> Save </button> + <button id="discard_script_but"> Cancel </button> + </div> + </div> + </td> + </tr> + </tbody> + </table> + </div> + </div> <button id="add_script_but" type="button"> Add script </button> </div> - <button id="import_but" style="margin-top: 40px;"> Import </button> - <div id="chbx_components_window" class="hide popup" position="absolute"> <div class="popup_frame"> - <ul id="chbx_components_ul"> - - </ul> + <div class="table_wrapper tight_table has_bottom_line has_upper_line"> + <div> + <table> + <tbody id="chbx_components_ul"> + </tbody> + </table> + </div> + </div> <button id="commit_bag_components_but"> Add </button> <button id="cancel_bag_components_but"> Cancel </button> </div> @@ -204,13 +361,20 @@ <div id="radio_components_window" class="hide popup" position="absolute"> <div class="popup_frame"> - <ul id="radio_components_ul"> - - <li id="radio_component_none_li"> - <input id="radio_component_none_input" type="radio" style="display: inline;" name="page_components"></input> - <span>(None)</span> - </li> - </ul> + <div class="table_wrapper tight_table always_scrollbar has_bottom_line has_upper_line"> + <div> + <table> + <tbody id="radio_components_ul"> + <tr id="radio_component_none_li"> + <td> + <input id="radio_component_none_input" type="radio" style="display: inline;" name="page_components"></input> + <span>(None)</span> + </td> + </tr> + </tbody> + </table> + </div> + </div> <button id="commit_page_components_but"> Choose </button> <button id="cancel_page_components_but"> Cancel </button> </div> @@ -218,6 +382,7 @@ <div id="import_window" class="hide popup" position="absolute"> <div class="popup_frame"> + <h2> Settings import </h2> <IMPORT html/import_frame.html /> </div> </div> diff --git a/html/options_main.js b/html/options_main.js index 830c860..f8faf9b 100644 --- a/html/options_main.js +++ b/html/options_main.js @@ -1,5 +1,7 @@ /** - * Hachette HTML options page main script + * This file is part of Haketilo. + * + * Function: Settings page logic. * * Copyright (C) 2021 Wojtek Kosior * Redistribution terms are gathered in the `copyright' file. @@ -13,24 +15,36 @@ * IMPORT list_prefixes * IMPORT nice_name * IMPORT parse_json_with_schema + * IMPORT get_template * IMPORT by_id * IMPORT matchers * IMPORT get_import_frame + * IMPORT init_default_policy_dialog * IMPORTS_END */ var storage; -const item_li_template = by_id("item_li_template"); -const bag_component_li_template = by_id("bag_component_li_template"); -const chbx_component_li_template = by_id("chbx_component_li_template"); -const radio_component_li_template = by_id("radio_component_li_template"); +const item_li_template = get_template("item_li"); +const bag_component_li_template = get_template("bag_component_li"); +const chbx_component_li_template = get_template("chbx_component_li"); +const radio_component_li_template = get_template("radio_component_li"); /* Make sure they are later cloned without id. */ item_li_template.removeAttribute("id"); bag_component_li_template.removeAttribute("id"); chbx_component_li_template.removeAttribute("id"); radio_component_li_template.removeAttribute("id"); +function list_set_scrollbar(list_elem) +{ + const op = ((list_elem.children.length === 1 && + list_elem.children[0].classList.contains("hide")) || + list_elem.children.length < 1) ? "remove" : "add"; + while (!list_elem.classList.contains("table_wrapper")) + list_elem = list_elem.parentElement; + list_elem.classList[op]("always_scrollbar"); +} + function item_li_id(prefix, item) { return `li_${prefix}_${item}`; @@ -65,11 +79,17 @@ function add_li(prefix, item, at_the_end=false) continue; ul.ul.insertBefore(li, element); - return; + break; } } + if (!li.parentElement) { + if (ul.work_li !== ul.ul.lastElementChild) + ul.ul.appendChild(li); + else + ul.work_li.before(li); + } - ul.ul.appendChild(li); + list_set_scrollbar(ul.ul); } const chbx_components_ul = by_id("chbx_components_ul"); @@ -97,12 +117,13 @@ function add_chbx_li(prefix, name) li.setAttribute("data-prefix", prefix); li.setAttribute("data-name", name); - let chbx = li.firstElementChild; + let chbx = li.firstElementChild.firstElementChild; let span = chbx.nextElementSibling; span.textContent = nice_name(prefix, name); chbx_components_ul.appendChild(li); + list_set_scrollbar(chbx_components_ul); } var radio_component_none_li = by_id("radio_component_none_li"); @@ -117,12 +138,13 @@ function add_radio_li(prefix, name) li.setAttribute("data-prefix", prefix); li.setAttribute("data-name", name); - let radio = li.firstElementChild; + let radio = li.firstElementChild.firstElementChild; let span = radio.nextElementSibling; span.textContent = nice_name(prefix, name); - radio_components_ul.insertBefore(li, radio_component_none_li); + radio_component_none_li.before(li); + list_set_scrollbar(radio_components_ul); } /* Used to reset edited repo. */ @@ -137,6 +159,7 @@ function work_repo_li_data(ul) return [ul.work_name_input.value, {}]; } +const allow_native_scripts_container = by_id("allow_native_scripts_container"); const page_payload_span = by_id("page_payload"); function set_page_components(components) @@ -144,12 +167,14 @@ function set_page_components(components) if (components === undefined) { page_payload_span.setAttribute("data-payload", "no"); page_payload_span.textContent = "(None)"; + allow_native_scripts_container.classList.remove("form_disabled"); } else { page_payload_span.setAttribute("data-payload", "yes"); let [prefix, name] = components; page_payload_span.setAttribute("data-prefix", prefix); page_payload_span.setAttribute("data-name", name); page_payload_span.textContent = nice_name(prefix, name); + allow_native_scripts_container.classList.add("form_disabled"); } } @@ -190,6 +215,13 @@ function work_page_li_data(ul) const empty_bag_component_li = by_id("empty_bag_component_li"); var bag_components_ul = by_id("bag_components_ul"); +function remove_bag_component_entry(entry) +{ + const list = entry.parentElement; + entry.remove(); + list_set_scrollbar(list); +} + /* Used to construct and update components list of edited bag. */ function add_bag_components(components) { @@ -198,15 +230,17 @@ function add_bag_components(components) let li = bag_component_li_template.cloneNode(true); li.setAttribute("data-prefix", prefix); li.setAttribute("data-name", name); + let span = li.firstElementChild; span.textContent = nice_name(prefix, name); let remove_but = span.nextElementSibling; - remove_but.addEventListener("click", () => - bag_components_ul.removeChild(li)); + remove_but.addEventListener("click", + () => remove_bag_component_entry(li)); bag_components_ul.appendChild(li); } bag_components_ul.appendChild(empty_bag_component_li); + list_set_scrollbar(bag_components_ul); } /* Used to reset edited bag. */ @@ -218,8 +252,7 @@ function reset_work_bag_li(ul, item, components) let old_components_ul = bag_components_ul; bag_components_ul = old_components_ul.cloneNode(false); - ul.work_li.insertBefore(bag_components_ul, old_components_ul); - ul.work_li.removeChild(old_components_ul); + old_components_ul.replaceWith(bag_components_ul); add_bag_components(components); } @@ -227,8 +260,7 @@ function reset_work_bag_li(ul, item, components) /* Used to get edited bag data for saving. */ function work_bag_li_data(ul) { - let components_ul = ul.work_name_input.nextElementSibling; - let component_li = components_ul.firstElementChild; + let component_li = bag_components_ul.firstElementChild; let components = []; @@ -284,6 +316,8 @@ function cancel_work(prefix) } ul.work_li.classList.add("hide"); + ul.ul.append(ul.work_li); + list_set_scrollbar(ul.ul); ul.state = UL_STATE.IDLE; } @@ -322,6 +356,7 @@ function edit_item(prefix, item) ul.ul.insertBefore(ul.work_li, li); ul.ul.removeChild(li); ul.work_li.classList.remove("hide"); + list_set_scrollbar(ul.ul); ul.state = UL_STATE.EDITING_ENTRY; ul.edited_item = item; @@ -379,6 +414,7 @@ function add_new_item(prefix, name) ul.reset_work_li(ul); ul.work_li.classList.remove("hide"); ul.ul.appendChild(ul.work_li); + list_set_scrollbar(ul.ul); if (name !== undefined) ul.work_name_input.value = name; @@ -393,7 +429,7 @@ function bag_components() radio_components_window.classList.add("hide"); for (let li of chbx_components_ul.children) { - let chbx = li.firstElementChild; + let chbx = li.firstElementChild.firstElementChild; chbx.checked = false; } } @@ -403,7 +439,7 @@ function commit_bag_components() let selected = []; for (let li of chbx_components_ul.children) { - let chbx = li.firstElementChild; + let chbx = li.firstElementChild.firstElementChild; if (!chbx.checked) continue; @@ -431,10 +467,11 @@ function page_components() let [prefix, item] = components; let li = by_id(radio_li_id(prefix, item)); + if (li === null) radio_component_none_input.checked = false; else - li.firstElementChild.checked = true; + li.firstElementChild.firstElementChild.checked = true; } function commit_page_components() @@ -442,7 +479,7 @@ function commit_page_components() let components = null; for (let li of radio_components_ul.children) { - let radio = li.firstElementChild; + let radio = li.firstElementChild.firstElementChild; if (!radio.checked) continue; @@ -603,6 +640,8 @@ async function initialize_import_facility() import_frame = await get_import_frame(); import_frame.onclose = hide_import_window; + import_frame.style_table("has_bottom_line", "always_scrollbar", + "has_upper_line", "tight_table"); } /* @@ -637,6 +676,8 @@ function jump_to_item(url_with_item) async function main() { + init_default_policy_dialog(); + storage = await get_remote_storage(); for (let prefix of list_prefixes) { @@ -709,6 +750,7 @@ function handle_change(change) for (let [components_ul, id_creator] of uls_creators) { let li = by_id(id_creator(change.prefix, change.item)); components_ul.removeChild(li); + list_set_scrollbar(components_ul); } } diff --git a/html/reset.css b/html/reset.css new file mode 100644 index 0000000..dab51cd --- /dev/null +++ b/html/reset.css @@ -0,0 +1,49 @@ +/* http://meyerweb.com/eric/tools/css/reset/ + v2.0 | 20110126 + Copyright 2008,2011 Eric A. Meyer + Redistribution terms are gathered in the `copyright' file. +*/ + +html, body, div, span, applet, object, iframe, +h1, h2, h3, h4, h5, h6, p, blockquote, pre, +a, abbr, acronym, address, big, cite, code, +del, dfn, em, img, ins, kbd, q, s, samp, +small, strike, strong, sub, sup, tt, var, +b, u, i, center, +dl, dt, dd, ol, ul, li, +fieldset, form, label, legend, +table, caption, tbody, tfoot, thead, tr, th, td, +article, aside, canvas, details, embed, +figure, figcaption, footer, header, hgroup, +menu, nav, output, ruby, section, summary, +time, mark, audio, video { + margin: 0; + padding: 0; + border: 0; + font-size: 100%; + font: inherit; + vertical-align: baseline; +} +/* HTML5 display-role reset for older browsers */ +article, aside, details, figcaption, figure, +footer, header, hgroup, menu, nav, section { + display: block; +} +body { + line-height: 1; +} +ol, ul { + list-style: none; +} +blockquote, q { + quotes: none; +} +blockquote:before, blockquote:after, +q:before, q:after { + content: ''; + content: none; +} +table { + border-collapse: collapse; + border-spacing: 0; +} diff --git a/html/table.css b/html/table.css new file mode 100644 index 0000000..36f88bb --- /dev/null +++ b/html/table.css @@ -0,0 +1,45 @@ +.table_wrapper { + display: block; + background-color: #f0f0f0; + margin: 6px 0; +} + +.table_wrapper table { + border-collapse: unset; + width: 100%; +} + +.table_wrapper.tight_table, +.table_wrapper.tight_table>*, +.table_wrapper.tight_table>*>table { + width: -moz-min-content; + width: min-content; +} + +tr:nth-child(odd) { + background-color: #e5e5e5; +} + +td { + vertical-align: middle; + min-width: fit-content; + min-width: -moz-fit-content; +} + +.tight_table td { + width: 1%; +} + +td:first-child { + padding: 3px 10px 6px; +} + +.tight_table td:first-child { + width: 100%; +} + +td>div.button { + margin-right: 4px; + white-space: nowrap; + float: right; +} diff --git a/icons/hachette.svg b/icons/haketilo.svg index 6e8948d..6e8948d 100644 --- a/icons/hachette.svg +++ b/icons/haketilo.svg diff --git a/icons/hachette128.png b/icons/haketilo128.png Binary files differindex 18816e9..18816e9 100644 --- a/icons/hachette128.png +++ b/icons/haketilo128.png diff --git a/icons/hachette16.png b/icons/haketilo16.png Binary files differindex 182ede5..182ede5 100644 --- a/icons/hachette16.png +++ b/icons/haketilo16.png diff --git a/icons/hachette32.png b/icons/haketilo32.png Binary files differindex ffaa84b..ffaa84b 100644 --- a/icons/hachette32.png +++ b/icons/haketilo32.png diff --git a/icons/hachette48.png b/icons/haketilo48.png Binary files differindex 1ffcd38..1ffcd38 100644 --- a/icons/hachette48.png +++ b/icons/haketilo48.png diff --git a/icons/hachette64.png b/icons/haketilo64.png Binary files differindex a02abb0..a02abb0 100644 --- a/icons/hachette64.png +++ b/icons/haketilo64.png diff --git a/licenses/agpl-3.0.txt b/licenses/agpl-3.0.txt index 5908d37..46b1055 100644 --- a/licenses/agpl-3.0.txt +++ b/licenses/agpl-3.0.txt @@ -2,7 +2,7 @@ GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 -Copyright © 2007 Free Software Foundation, Inc. <http://fsf.org/> +Copyright © 2007 Free Software Foundation, Inc. <http://fsf.org/> Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. @@ -195,20 +195,20 @@ You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: - • a) The work must carry prominent notices stating that you modified it, + • a) The work must carry prominent notices stating that you modified it, and giving a relevant date. - • b) The work must carry prominent notices stating that it is released + • b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". - • c) You must license the entire work, as a whole, under this License to + • c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. - • d) If the work has interactive user interfaces, each must display + • d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. @@ -228,11 +228,11 @@ You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: - • a) Convey the object code in, or embodied in, a physical product + • a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. - • b) Convey the object code in, or embodied in, a physical product + • b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give @@ -242,11 +242,11 @@ Corresponding Source under the terms of this License, in one of these ways: for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. - • c) Convey individual copies of the object code with a copy of the + • c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. - • d) Convey the object code by offering access from a designated place + • d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the @@ -258,7 +258,7 @@ Corresponding Source under the terms of this License, in one of these ways: Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. - • e) Convey the object code using peer-to-peer transmission, provided you + • e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. @@ -334,19 +334,19 @@ Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: - • a) Disclaiming warranty or limiting liability differently from the + • a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or - • b) Requiring preservation of specified reasonable legal notices or + • b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or - • c) Prohibiting misrepresentation of the origin of that material, or + • c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or - • d) Limiting the use for publicity purposes of names of licensors or + • d) Limiting the use for publicity purposes of names of licensors or authors of the material; or - • e) Declining to grant rights under trademark law for use of some trade + • e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or - • f) Requiring indemnification of licensors and authors of that material + • f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those diff --git a/manifest.json b/manifest.json index 328b27f..7b4cb26 100644 --- a/manifest.json +++ b/manifest.json @@ -1,18 +1,20 @@ +// This is the manifest file of Haketilo. +// // Copyright (C) 2021 Wojtek Kosior // Redistribution terms are gathered in the `copyright' file. { "manifest_version": 2, - "name": "Hachette", - "short_name": "Hachette", - "version": "0.0.1",_CHROMIUM_KEY_ + "name": "Haketilo", + "short_name": "Haketilo", + "version": "0.1", "author": "various", - "description": "Kill the web&js",_GECKO_APPLICATIONS_ + "description": "Control your \"Web\" browsing.",_GECKO_APPLICATIONS_ "icons":{ - "128": "icons/hachette128.png", - "64": "icons/hachette64.png", - "48": "icons/hachette48.png", - "32": "icons/hachette32.png", - "16": "icons/hachette16.png" + "128": "icons/haketilo128.png", + "64": "icons/haketilo64.png", + "48": "icons/haketilo48.png", + "32": "icons/haketilo32.png", + "16": "icons/haketilo16.png" }, "permissions": [ "contextMenus", @@ -23,26 +25,26 @@ "sessions", "storage", "tabs", - "<all_urls>" + "<all_urls>", + "unlimitedStorage" ], "browser_action": { "browser_style": true, "default_icon": { - "128": "icons/hachette128.png", - "64": "icons/hachette64.png", - "48": "icons/hachette48.png", - "32": "icons/hachette32.png", - "16": "icons/hachette16.png" + "128": "icons/haketilo128.png", + "64": "icons/haketilo64.png", + "48": "icons/haketilo48.png", + "32": "icons/haketilo32.png", + "16": "icons/haketilo16.png" }, - "default_title": "Hachette", + "default_title": "Haketilo", "default_popup": "html/display-panel.html" }, "options_ui": { "page": "html/options.html", "open_in_tab": true - }, - "web_accessible_resources": [ - ], + }_CHROMIUM_UPDATE_URL_, + "web_accessible_resources": ["dummy"], "background": { "persistent": true, "scripts": [_BGSCRIPTS_] diff --git a/process_html_file.sh b/process_html_file.sh index 1ed0295..2f58cbf 100755 --- a/process_html_file.sh +++ b/process_html_file.sh @@ -12,7 +12,7 @@ FILE="$1" FILEKEY=$(sanitize "$FILE") if [ "x$(map_get HTML_FILENAMES $FILEKEY)" = "xyes" ]; then - errcho "import loop on $FILE" + printf 'import loop on %s\n' "$FILE" >&2 exit 1 fi diff --git a/re-generate_icons.sh b/re-generate_icons.sh index ba0c28a..e557ad0 100755 --- a/re-generate_icons.sh +++ b/re-generate_icons.sh @@ -4,5 +4,5 @@ # Redistribution terms are gathered in the `copyright' file. for SIZE in 128 64 48 32 16; do - inkscape -z -e icons/hachette$SIZE.png -w $SIZE -h $SIZE icons/hachette.svg + inkscape -z -e icons/haketilo$SIZE.png -w $SIZE -h $SIZE icons/haketilo.svg done diff --git a/shell_utils.sh b/shell_utils.sh index 95e0d4e..6d4cc76 100644 --- a/shell_utils.sh +++ b/shell_utils.sh @@ -3,15 +3,8 @@ # This file is meant to be sourced in sh. -ENDL=" -" - -errcho() { - echo "$@" >&2 -} - map_set_instr() { - echo "$1__$2='$3'" + printf "%s__%s='%s'" "$1" "$2" "$3" } map_set() { @@ -23,11 +16,11 @@ map_set_export() { } map_get() { - eval "echo \"\$$1__$2\"" + eval "printf %s \"\$$1__$2\"" } map_del_instr() { - echo "unset $1__$2" + printf 'unset %s__%s' "$1" "$2" } map_del() { @@ -35,5 +28,18 @@ map_del() { } sanitize() { - echo "$1" | tr /.- _ + printf %s "$1" | tr /.- _ +} + +escape_regex_special() { + printf %s "$1" | sed 's/\([]\.*?{},()[-]\)/\\\1/g' +} + +# Note: We don't actually parse JSON. We extract needed keys with sed regexes +# which does not work in the general case but is sufficient for now. +get_json_key() { + local KEY_REG="$(escape_regex_special "$1")" + printf %s "$2" | + sed 's/\(.*"'"$KEY_REG"'"[[:space:]]*:[[:space:]]*"\([^"]*\)"\)\?.*/\2/' | + grep . | head -1 } diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..2b351bb --- /dev/null +++ b/test/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: CC0-1.0 +# Copyright (C) 2021 Wojtek Kosior diff --git a/test/__main__.py b/test/__main__.py new file mode 100644 index 0000000..c3437ea --- /dev/null +++ b/test/__main__.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +""" +Run a Firefox-type browser with WebDriver attached and Python console open +""" + +# This file is part of Haketilo. +# +# Copyright (C) 2021 jahoti <jahoti@tilde.team> +# Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this +# file's license. Although I request that you do not make use this code +# in a proprietary program, I am not going to enforce this in court. + +import sys +import time +import code + +from .server import do_an_internet +from .misc_constants import * +from .profiles import firefox_safe_mode + +def fail(msg, error_code): + print('Error:', msg) + print('Usage:', sys.argv[0], '[certificates_directory] [proxy_port]') + sys.exit(error_code) + +certdir = Path(sys.argv[1]).resolve() if len(sys.argv) > 1 else default_cert_dir +if not certdir.is_dir(): + fail('selected certificate directory does not exist.', 2) + +port = sys.argv[2] if len(sys.argv) > 2 else str(default_proxy_port) +if not port.isnumeric(): + fail('port must be an integer.', 3) + +httpd = do_an_internet(certdir, int(port)) +driver = firefox_safe_mode(proxy_port=int(port)) + +print("You can now control the browser through 'driver' object") + +code.InteractiveConsole(locals=globals()).interact() + +driver.quit() +httpd.shutdown() diff --git a/test/data/pages/gotmyowndomain.html b/test/data/pages/gotmyowndomain.html index daa9ab7..42c26cc 100644 --- a/test/data/pages/gotmyowndomain.html +++ b/test/data/pages/gotmyowndomain.html @@ -1,9 +1,30 @@ +<!DOCTYPE html> +<!-- + SPDX-License-Identifier: AGPL-3.0-or-later + + Sample testig page + + This file is part of Haketilo. + + Copyright (C) 2021 jahoti <jahoti@tilde.team> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + --> <html> <head> <meta name=charset value="latin1"> - <title> - Schrodinger's Document - </title> + <title>Schrodinger's Document</title> </head> <body> A nice, simple page for testing. diff --git a/test/data/pages/gotmyowndomain_https.html b/test/data/pages/gotmyowndomain_https.html index ec2aa1f..95c0be4 100644 --- a/test/data/pages/gotmyowndomain_https.html +++ b/test/data/pages/gotmyowndomain_https.html @@ -1,9 +1,30 @@ +<!DOCTYPE html> +<!-- + SPDX-License-Identifier: AGPL-3.0-or-later + + Sample testig page to serve over HTTPS + + This file is part of Haketilo. + + Copyright (C) 2021 jahoti <jahoti@tilde.team> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + --> <html> <head> <meta name=charset value="latin1"> - <title> - Schrodinger's Document - </title> + <title>Schrodinger's Document</title> </head> <body> A nice, simple page for testing (using HTTPS). diff --git a/test/default_profiles/icecat_empty/extensions.json b/test/default_profiles/icecat_empty/extensions.json new file mode 100644 index 0000000..5f74ff3 --- /dev/null +++ b/test/default_profiles/icecat_empty/extensions.json @@ -0,0 +1 @@ +{"schemaVersion":25,"addons":[{"id":"jid1-KtlZuoiikVfFew@jetpack","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/jid1-KtlZuoiikVfFew@jetpack"},{"id":"uBlock0@raymondhill.net","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/uBlock0@raymondhill.net.xpi"},{"id":"SubmitMe@0xbeef.coffee","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/SubmitMe@0xbeef.coffee"},{"id":"FreeUSPS@0xbeef.coffee","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/FreeUSPS@0xbeef.coffee"},{"id":"tortm-browser-button@jeremybenthum","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/tortm-browser-button@jeremybenthum"},{"id":"tprb.addon@searxes.danwin1210.me","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/tprb.addon@searxes.danwin1210.me"},{"id":"SimpleSumOfUs@0xbeef.coffee","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/SimpleSumOfUs@0xbeef.coffee"}]}
\ No newline at end of file diff --git a/test/gorilla.py b/test/gorilla.py deleted file mode 100755 index 5bf64f5..0000000 --- a/test/gorilla.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2021 jahoti <jahoti@tilde.team> -# Licensing information is collated in the `copyright` file - -""" -A partial re-implementation of Hydrilla in Python, for testing purposes. - -This will eventually be replaced with a build of the actual thing. -""" - -import json, os, sys - -def load_db(path): - DB = {'script': {}, 'bag': {}, 'page': {}} - if path[-1] != os.sep: - path += os.sep - - DB['path'] = path - for file in os.listdir(path): - subdir = path + file - - if (os.st.S_IFMT(os.stat(subdir).st_mode) & os.st.S_IFDIR) == 0: - continue - - with open(subdir + os.sep + 'index.json') as f: - data = json.load(f) - - type_, data['file'] = data['type'], file - name_key = 'pattern' if type_ == 'page' else 'name' - - DB[type_][data[name_key]] = data - del data['type'], data[name_key] - if type_ == 'script': - with open(path + file + os.sep + data['location']) as f: - data['text'] = f.read() - - return DB - - -def mkhachette(patterns): - patterns = set(patterns) - if '*' in patterns: - patterns.remove('*') - patterns.update(DB['page'].keys()) - - scripts, bags, pages, new_bags = {}, {}, {}, [] - for pattern in patterns: - pages[pattern] = data = DB['page'][pattern] - type_, name = data['payload'] - if type_ == 'script': - scripts[name] = DB['script'][name] - else: - new_bags.append(name) - - while new_bags: - name = new_bags.pop(0) - if name in bags: - continue - - bags[name] = data = DB['bag'][name]['components'] - for type_, name in data: - if type_ == 'script': - scripts[name] = DB['script'][name] - else: - new_bags.append(name) - - items, path = [], DB['path'] - for script, data in scripts.items(): - item = {'hash': data['sha256']} - with open(path + data['file'] + os.sep + data['location']) as f: - item['text'] = f.read() - - items.append({'s' + script: item}) - - for bag, data in bags.items(): - items.append({'b' + bag: [[type_[0], name] for type_, name in data]}) - - for page, data in pages.items(): - type_, name = data['payload'] - items.append({'p' + page: {'components': [type_[0], name]}}) - - return items - - -if __name__ == '__main__': - if len(sys.argv) < 3 or not os.path.isdir(sys.argv[1]): - sys.stderr.write('Usage: %s [hydrilla content path] (page pattern (page pattern (...)))' % sys.argv[0]) - sys.exit(1) - - DB = load_db(sys.argv[1]) - print(json.dumps(mkhachette(sys.argv[2:]))) diff --git a/test/init.sh b/test/init.sh deleted file mode 100755 index c0e7647..0000000 --- a/test/init.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2015, inaz2 -# Copyright (C) 2021 jahoti <jahoti@tilde.team> -# Licensing information is collated in the `copyright` file - -# Initialize the root certificate for the tests proxy server -# Make sure this is run in the directory where they will be put! - - -if [ -n "$1" ]; then - cd "$1" -fi -openssl genrsa -out ca.key 2048 -openssl genrsa -out cert.key 2048 -openssl req -new -x509 -days 3650 -key ca.key -out ca.crt -subj "/CN=Hachette Test" diff --git a/test/misc_constants.py b/test/misc_constants.py index c6c134f..22432a6 100644 --- a/test/misc_constants.py +++ b/test/misc_constants.py @@ -1,10 +1,46 @@ -# Copyright (C) 2021 jahoti <jahoti@tilde.team> -# Licensing information is collated in the `copyright` file +# SPDX-License-Identifier: AGPL-3.0-or-later """ Miscellaneous data that were found useful """ +# This file is part of Haketilo. +# +# Copyright (C) 2021 jahoti <jahoti@tilde.team> +# Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this +# file's license. Although I request that you do not make use this code +# in a proprietary program, I am not going to enforce this in court. + +from pathlib import Path + +here = Path(__file__).resolve().parent + +default_firefox_binary = '/usr/lib/icecat/icecat' +# The browser might be loading some globally-installed add-ons by default. They +# could interfere with the tests, so we'll disable all of them. +default_clean_profile_dir = here / 'default_profile' / 'icecat_empty' + +default_proxy_host = '127.0.0.1' +default_proxy_port = 1337 + +default_cert_dir = here / 'certs' + mime_types = { "7z": "application/x-7z-compressed", "oga": "audio/ogg", "abw": "application/x-abiword", "ogv": "video/ogg", @@ -22,7 +58,7 @@ mime_types = { "js": "text/javascript", "wav": "audio/wav", "jpeg": "image/jpeg", "weba": "audio/webm", "jpg": "image/jpeg", "webm": "video/webm", - "json": "application/json", "woff": "font/woff", + "json": "application/json", "woff": "font/woff", "mjs": "text/javascript", "woff2": "font/woff2", "mp3": "audio/mpeg", "xhtml": "application/xhtml+xml", "mp4": "video/mp4", "zip": "application/zip", diff --git a/test/profiles.py b/test/profiles.py new file mode 100755 index 0000000..d6a4efc --- /dev/null +++ b/test/profiles.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +""" +Browser profiles and Selenium driver initialization +""" + +# This file is part of Haketilo. +# +# Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this file's +# license. Although I request that you do not make use this code in a +# proprietary program, I am not going to enforce this in court. + +from selenium import webdriver +from selenium.webdriver.firefox.options import Options +import time + +from .misc_constants import * + +def set_profile_proxy(profile, proxy_host, proxy_port): + # proxy type 1 designates "manual" + profile.set_preference('network.proxy.type', 1) + profile.set_preference('network.proxy.no_proxies_on', '') + profile.set_preference('network.proxy.share_proxy_settings', True) + + for proto in ['http', 'ftp', 'socks', 'ssl']: + profile.set_preference(f'network.proxy.{proto}', proxy_host) + profile.set_preference(f'network.proxy.{proto}_port', proxy_port) + profile.set_preference(f'network.proxy.backup.{proto}', '') + profile.set_preference(f'network.proxy.backup.{proto}_port', 0) + +def set_profile_console_logging(profile): + profile.set_preference('devtools.console.stdout.content', True) + +def firefox_safe_mode(firefox_binary=default_firefox_binary, + proxy_host=default_proxy_host, + proxy_port=default_proxy_port): + profile = webdriver.FirefoxProfile() + set_profile_proxy(profile, proxy_host, proxy_port) + set_profile_console_logging(profile) + + options = Options() + options.add_argument('--safe-mode') + + return webdriver.Firefox(options=options, firefox_profile=profile, + firefox_binary=firefox_binary) + +def firefox_with_profile(firefox_binary=default_firefox_binary, + profile_dir=default_clean_profile_dir, + proxy_host=default_proxy_host, + proxy_port=default_proxy_port): + profile = webdriver.FirefoxProfile(profile_dir) + set_profile_proxy(profile, proxy_host, proxy_port) + set_profile_console_logging(profile) + + return webdriver.Firefox(firefox_profile=profile, + firefox_binary=firefox_binary) diff --git a/test/proxy_core.py b/test/proxy_core.py index dd4225d..d31302a 100644 --- a/test/proxy_core.py +++ b/test/proxy_core.py @@ -1,74 +1,141 @@ -# Copyright (c) 2015, inaz2 -# Copyright (C) 2021 jahoti <jahoti@tilde.team> -# Licensing information is collated in the `copyright` file +# SPDX-License-Identifier: BSD-3-Clause """ -The core for a "virtual network" proxy - -Be sure to set certdir to your intended certificates directory before running. +The core for a "virtual network" proxy. """ -import os, socket, ssl, subprocess, sys, threading, time +# This file is part of Haketilo. +# +# Copyright (c) 2015, inaz2 +# Copyright (C) 2021 jahoti <jahoti@tilde.team> +# Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org> +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of proxy2 nor the names of its contributors may be used to +# endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this file's +# license. Although I request that you do not make use this code in a way +# incompliant with the license, I am not going to enforce this in court. + +from pathlib import Path +import socket, ssl, subprocess, sys, threading from http.server import HTTPServer, BaseHTTPRequestHandler from socketserver import ThreadingMixIn -gen_cert_req, lock = 'openssl req -new -key %scert.key -subj /CN=%s', threading.Lock() -sign_cert_req = 'openssl x509 -req -days 3650 -CA %sca.crt -CAkey %sca.key -set_serial %d -out %s' - -def popen(command, *args, **kwargs): - return subprocess.Popen((command % args).split(' '), **kwargs) +lock = threading.Lock() class ProxyRequestHandler(BaseHTTPRequestHandler): - """Handles a network request made to the proxy""" - def log_error(self, format, *args): - # suppress "Request timed out: timeout('timed out',)" - if isinstance(args[0], socket.timeout): - return - - self.log_message(format, *args) - - def do_CONNECT(self): - hostname = self.path.split(':')[0] - certpath = '%s%s.crt' % (certdir, hostname if hostname != 'ca' else 'CA') - - with lock: - if not os.path.isfile(certpath): - p1 = popen(gen_cert_req, certdir, hostname, stdout=subprocess.PIPE).stdout - popen(sign_cert_req, certdir, certdir, time.time() * 1000, certpath, stdin=p1, stderr=subprocess.PIPE).communicate() - - self.send_response(200) - self.end_headers() - - self.connection = ssl.wrap_socket(self.connection, keyfile=certdir+'cert.key', certfile=certpath, server_side=True) - self.rfile = self.connection.makefile('rb', self.rbufsize) - self.wfile = self.connection.makefile('wb', self.wbufsize) - - self.close_connection = int(self.headers.get('Proxy-Connection', '').lower() == 'close') - - def proxy(self): - content_length = int(self.headers.get('Content-Length', 0)) - req_body = self.rfile.read(content_length) if content_length else None - - if self.path[0] == '/': - if isinstance(self.connection, ssl.SSLSocket): - self.path = 'https://%s%s' % (self.headers['Host'], self.path) - else: - self.path = 'http://%s%s' % (self.headers['Host'], self.path) - - self.handle_request(req_body) - - do_OPTIONS = do_DELETE = do_PUT = do_HEAD = do_POST = do_GET = proxy - - def handle_request(self, req_body): - pass + """ + Handles a network request made to the proxy. Configures SSL encryption when + needed. + """ + def __init__(self, *args, **kwargs): + """ + Initialize self. Uses the same arguments as + http.server.BaseHTTPRequestHandler's constructor but also expect a + `certdir` keyword argument with appropriate path. + """ + self.certdir = Path(kwargs.pop('certdir')).resolve() + super().__init__(*args, **kwargs) + + def log_error(self, *args, **kwargs): + """ + Like log_error in http.server.BaseHTTPRequestHandler but suppresses + "Request timed out: timeout('timed out',)". + """ + if not isinstance(args[0], socket.timeout): + super().log_error(*args, **kwargs) + + def get_cert(self, hostname): + """ + If needed, generate a signed x509 certificate for `hostname`. Return + paths to certificate's key file and to certificate itself in a tuple. + """ + root_keyfile = self.certdir / 'rootCA.key' + root_certfile = self.certdir / 'rootCA.pem' + keyfile = self.certdir / 'site.key' + certfile = self.certdir / f'{hostname}.crt' + + with lock: + requestfile = self.certdir / f'{hostname}.csr' + if not certfile.exists(): + subprocess.run([ + 'openssl', 'req', '-new', '-key', str(keyfile), + '-subj', f'/CN={hostname}', '-out', str(requestfile) + ], check=True) + subprocess.run([ + 'openssl', 'x509', '-req', '-in', str(requestfile), + '-CA', str(root_certfile), '-CAkey', str(root_keyfile), + '-CAcreateserial', '-out', str(certfile), '-days', '1024' + ], check=True) + + return keyfile, certfile + + def do_CONNECT(self): + """Wrap the connection with SSL using on-demand signed certificate.""" + hostname = self.path.split(':')[0] + sslargs = {'server_side': True} + sslargs['keyfile'], sslargs['certfile'] = self.get_cert(hostname) + + self.send_response(200) + self.end_headers() + + self.connection = ssl.wrap_socket(self.connection, **sslargs) + self.rfile = self.connection.makefile('rb', self.rbufsize) + self.wfile = self.connection.makefile('wb', self.wbufsize) + + connection_header = self.headers.get('Proxy-Connection', '').lower() + self.close_connection = int(connection_header == 'close') + + def do_GET(self): + content_length = int(self.headers.get('Content-Length', 0)) + req_body = self.rfile.read(content_length) if content_length else None + + if self.path[0] == '/': + secure = 's' if isinstance(self.connection, ssl.SSLSocket) else '' + self.path = f'http{secure}://{self.headers["Host"]}{self.path}' + + self.handle_request(req_body) + + do_OPTIONS = do_DELETE = do_PUT = do_HEAD = do_POST = do_GET + + def handle_request(self, req_body): + """Default handler that does nothing. Please override.""" + pass class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): - """The actual proxy server""" - address_family, daemon_threads = socket.AF_INET6, True - - def handle_error(self, request, client_address): - # suppress socket/ssl related errors - cls, e = sys.exc_info()[:2] - if not (cls is socket.error or cls is ssl.SSLError): - return HTTPServer.handle_error(self, request, client_address) + """The actual proxy server""" + address_family, daemon_threads = socket.AF_INET6, True + + def handle_error(self, request, client_address): + """ + Like handle_error in http.server.HTTPServer but suppresses socket/ssl + related errors. + """ + cls, e = sys.exc_info()[:2] + if not (cls is socket.error or cls is ssl.SSLError): + return super().handle_error(request, client_address) diff --git a/test/script_loader.py b/test/script_loader.py new file mode 100644 index 0000000..15269c7 --- /dev/null +++ b/test/script_loader.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +""" +Loading of parts of Haketilo source for testing in browser +""" + +# This file is part of Haketilo. +# +# Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this file's +# license. Although I request that you do not make use this code in a +# proprietary program, I am not going to enforce this in court. + +from pathlib import Path +import subprocess, re + +from .misc_constants import * + +script_root = here.parent +awk_script = script_root / 'compute_scripts.awk' + +def make_relative_path(path): + path = Path(path) + + if path.is_absolute(): + path = path.relative_to(script_root) + + return path + +"""Used to ignore hidden files and emacs auto-save files.""" +script_name_regex = re.compile(r'^[^.#].*\.js$') + +def available_scripts(directory): + for script in directory.rglob('*.js'): + if script_name_regex.match(script.name): + yield script + +def wrapped_script(script_path, wrap_partially=True): + if script_path == 'exports_init.js': + with open(script_root / 'MOZILLA_exports_init.js') as script: + return script.read() + + command = 'partially_wrapped_code' if wrap_partially else 'wrapped_code' + awk_command = ['awk', '-f', str(awk_script), command, str(script_path)] + awk = subprocess.run(awk_command, stdout=subprocess.PIPE, cwd=script_root, + check=True) + + return awk.stdout.decode() + +def load_script(path, import_dirs): + """ + `path` and `import_dirs` are .js file path and a list of directory paths, + respectively. They may be absolute or specified relative to Haketilo's + project directory. + + Return a string containing script from `path` together with all other + scripts it depends. Dependencies are wrapped in the same way Haketilo's + build system wraps them, with imports properly satisfied. The main script + being loaded is wrapped partially - it also has its imports satisfied, but + its code is not placed inside an anonymous function, so the + """ + path = make_relative_path(path) + + import_dirs = [make_relative_path(dir) for dir in import_dirs] + available = [s for dir in import_dirs for s in available_scripts(dir)] + + awk = subprocess.run(['awk', '-f', str(awk_script), 'script_dependencies', + str(path), *[str(s) for s in available]], + stdout=subprocess.PIPE, cwd=script_root, check=True) + + to_load = awk.stdout.decode().split() + texts = [wrapped_script(path, wrap_partially=(i == len(to_load) - 1)) + for i, path in enumerate(to_load)] + + return '\n'.join(texts) diff --git a/test/server.py b/test/server.py index 58a84bd..6013955 100755 --- a/test/server.py +++ b/test/server.py @@ -1,101 +1,108 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2021 jahoti <jahoti@tilde.team> -# Licensing information is collated in the `copyright` file +# SPDX-License-Identifier: AGPL-3.0-or-later """ A modular "virtual network" proxy, wrapping the classes in proxy_core.py """ -import proxy_core +# This file is part of Haketilo. +# +# Copyright (C) 2021 jahoti <jahoti@tilde.team> +# Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this +# file's license. Although I request that you do not make use this code +# in a proprietary program, I am not going to enforce this in court. + +from pathlib import Path from urllib.parse import parse_qs -from misc_constants import * -from world_wide_library import catalog as internet - -class RequestHijacker(proxy_core.ProxyRequestHandler): - def handle_request(self, req_body): - path_components = self.path.split('?', maxsplit=1) - path = path_components[0] - try: - # Response format: (status_code, headers (dict. of strings), - # body as bytes or filename containing body as string) - if path in internet: - info = internet[path] - if type(info) == tuple: - status_code, headers, body_file = info - if type(body_file) == str: - if 'Content-Type' not in headers and '.' in body_file: - ext = body_file.rsplit('.', maxsplit=1)[-1] - if ext in mime_types: - headers['Content-Type'] = mime_types[ext] - - with open(body_file, mode='rb') as f: - body_file = f.read() - - else: - # A function to evaluate to get the response - get_params, post_params = {}, {} - if len(path_components) == 2: - get_params = parse_qs(path_components[1]) - - # Parse POST parameters; currently only supports - # application/x-www-form-urlencoded - if req_body: - post_params = parse_qs(req_body.encode()) - - status_code, headers, body_file = info(self.command, get_params, post_params) - if type(body_file) == str: - body_file = body_file.encode() - - if type(status_code) != int or status_code <= 0: - raise Exception('Invalid status code %r' % status_code) - - for header, header_value in headers.items(): - if type(header) != str: - raise Exception('Invalid header key %r' % header) - - elif type(header_value) != str: - raise Exception('Invalid header value %r' % header_value) - else: - status_code, headers = 404, {'Content-Type': 'text/plain'} - body_file = b'Handler for this URL not found.' - - except Exception as e: - status_code, headers, body_file = 500, {'Content-Type': 'text/plain'}, b'Internal Error:\n' + repr(e).encode() - - headers['Content-Length'] = str(len(body_file)) - self.send_response(status_code) - for header, header_value in headers.items(): - self.send_header(header, header_value) - - self.end_headers() - self.wfile.write(body_file) - - - -def do_an_internet(certdir, port): - """Start up the proxy/server""" - proxy_core.certdir = certdir - httpd = proxy_core.ThreadingHTTPServer(('', port), RequestHijacker) - httpd.serve_forever() - -if __name__ == '__main__': - import sys - def fail(msg, error_code): - print('Error:', msg) - print('Usage:', sys.argv[0], '[certificates directory] (port)') - sys.exit(error_code) - - if len(sys.argv) < 2: - fail('missing required argument "certificates directory".', 1) - - certdir = sys.argv[1] - if not proxy_core.os.path.isdir(certdir): - fail('selected certificate directory does not exist.', 2) - - port = sys.argv[2] if len(sys.argv) > 2 else '1337' - if not port.isnumeric(): - fail('port must be an integer.', 3) - - do_an_internet(certdir, int(port)) +from threading import Thread + +from .proxy_core import ProxyRequestHandler, ThreadingHTTPServer +from .misc_constants import * +from .world_wide_library import catalog as internet + +class RequestHijacker(ProxyRequestHandler): + def handle_request(self, req_body): + path_components = self.path.split('?', maxsplit=1) + path = path_components[0] + try: + # Response format: (status_code, headers (dict. of strings), + # body as bytes or filename containing body as string) + if path in internet: + info = internet[path] + if type(info) is tuple: + status_code, headers, body_file = info + resp_body = b'' + if body_file is not None: + if 'Content-Type' not in headers: + ext = body_file.suffix[1:] + if ext and ext in mime_types: + headers['Content-Type'] = mime_types[ext] + + with open(body_file, mode='rb') as f: + resp_body = f.read() + else: + # A function to evaluate to get the response + get_params, post_params = {}, {} + if len(path_components) == 2: + get_params = parse_qs(path_components[1]) + + # Parse POST parameters; currently only supports + # application/x-www-form-urlencoded + if req_body: + post_params = parse_qs(req_body.encode()) + + status_code, headers, resp_body = info(self.command, get_params, post_params) + if type(resp_body) == str: + resp_body = resp_body.encode() + + if type(status_code) != int or status_code <= 0: + raise Exception('Invalid status code %r' % status_code) + + for header, header_value in headers.items(): + if type(header) != str: + raise Exception('Invalid header key %r' % header) + + elif type(header_value) != str: + raise Exception('Invalid header value %r' % header_value) + else: + status_code, headers = 404, {'Content-Type': 'text/plain'} + resp_body = b'Handler for this URL not found.' + + except Exception as e: + status_code, headers, resp_body = 500, {'Content-Type': 'text/plain'}, b'Internal Error:\n' + repr(e).encode() + + headers['Content-Length'] = str(len(resp_body)) + self.send_response(status_code) + for header, header_value in headers.items(): + self.send_header(header, header_value) + + self.end_headers() + if resp_body: + self.wfile.write(resp_body) + +def do_an_internet(certdir=default_cert_dir, port=default_proxy_port): + """Start up the proxy/server""" + class RequestHijackerWithCertdir(RequestHijacker): + def __init__(self, *args, **kwargs): + super().__init__(*args, certdir=certdir, **kwargs) + + httpd = ThreadingHTTPServer(('', port), RequestHijackerWithCertdir) + Thread(target=httpd.serve_forever).start() + + return httpd diff --git a/test/unit/__init__.py b/test/unit/__init__.py new file mode 100644 index 0000000..2b351bb --- /dev/null +++ b/test/unit/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: CC0-1.0 +# Copyright (C) 2021 Wojtek Kosior diff --git a/test/unit/conftest.py b/test/unit/conftest.py new file mode 100644 index 0000000..6877b7a --- /dev/null +++ b/test/unit/conftest.py @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +""" +Common fixtures for Haketilo unit tests +""" + +# This file is part of Haketilo. +# +# Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this file's +# license. Although I request that you do not make use this code in a +# proprietary program, I am not going to enforce this in court. + +import pytest + +from ..profiles import firefox_safe_mode +from ..server import do_an_internet +from ..script_loader import load_script + +@pytest.fixture(scope="package") +def proxy(): + httpd = do_an_internet() + yield httpd + httpd.shutdown() + +@pytest.fixture(scope="package") +def driver(proxy): + with firefox_safe_mode() as driver: + yield driver + driver.quit() + +script_injecting_script = '''\ +/* + * Selenium by default executes scripts in some weird one-time context. We want + * separately-loaded scripts to be able to access global variables defined + * before, including those declared with `const` or `let`. To achieve that, we + * run our scripts by injecting them into the page inside a <script> tag. We use + * custom properties of the `window` object to communicate with injected code. + */ + +const script_elem = document.createElement('script'); +script_elem.textContent = arguments[0]; + +delete window.haketilo_selenium_return_value; +delete window.haketilo_selenium_exception; +window.returnval = (val => window.haketilo_selenium_return_value = val); +window.arguments = arguments[1]; + +document.body.append(script_elem); + +/* + * To ease debugging, we want this script to forward signal all exceptions from + * the injectee. + */ +try { + if (window.haketilo_selenium_exception !== false) + throw 'Error in injected script! Check your geckodriver.log!'; +} finally { + script_elem.remove(); +} + +return window.haketilo_selenium_return_value; +''' + +def _execute_in_page_context(driver, script, *args): + script = script + '\n;\nwindow.haketilo_selenium_exception = false;' + try: + return driver.execute_script(script_injecting_script, script, args) + except Exception as e: + import sys + lines = enumerate(script.split('\n'), 1) + for err_info in [('Failing script\n',), *lines]: + print(*err_info, file=sys.stderr) + + raise e from None + +@pytest.fixture(scope="package") +def execute_in_page(driver): + def do_execute(script, *args, **kwargs): + if 'page' in kwargs: + driver.get(kwargs['page']) + + return _execute_in_page_context(driver, script, args) + + yield do_execute + +@pytest.fixture(scope="package") +def load_into_page(driver): + def do_load(path, import_dirs, *args, **kwargs): + if 'page' in kwargs: + driver.get(kwargs['page']) + + _execute_in_page_context(driver, load_script(path, import_dirs), args) + + yield do_load diff --git a/test/unit/test_basic.py b/test/unit/test_basic.py new file mode 100644 index 0000000..cbe5c8c --- /dev/null +++ b/test/unit/test_basic.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - base +""" + +# This file is part of Haketilo +# +# Copyright (C) 2021, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest + +def test_driver(driver): + """ + A trivial test case that verifies mocked web pages served by proxy can be + accessed by the browser driven. + """ + for proto in ['http://', 'https://']: + driver.get(proto + 'gotmyowndoma.in') + element = driver.find_element_by_tag_name('title') + title = driver.execute_script('return arguments[0].innerText;', element) + assert "Schrodinger's Document" in title + +def test_script_loader(execute_in_page, load_into_page): + """ + A trivial test case that verifies Haketilo's .js files can be properly + loaded into a test page together with their dependencies. + """ + load_into_page('common/stored_types.js', ['common'], + page='https://gotmyowndoma.in') + + assert execute_in_page('returnval(TYPE_PREFIX.VAR);') == '_' diff --git a/test/unit/test_patterns.py b/test/unit/test_patterns.py new file mode 100644 index 0000000..4162fc0 --- /dev/null +++ b/test/unit/test_patterns.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - URL patterns +""" + +# This file is part of Haketilo +# +# Copyright (C) 2021, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest + +from ..script_loader import load_script + +@pytest.fixture(scope="session") +def patterns_code(): + yield load_script('common/patterns.js', ['common']) + +def test_regexes(execute_in_page, patterns_code): + """ + patterns.js contains regexes used for URL parsing. + Verify they work properly. + """ + execute_in_page(patterns_code, page='https://gotmyowndoma.in') + + valid_url = 'https://example.com/a/b?ver=1.2.3#heading2' + valid_url_rest = 'example.com/a/b?ver=1.2.3#heading2' + + # Test matching of URL protocol. + match = execute_in_page('returnval(proto_regex.exec(arguments[0]));', + valid_url) + assert match + assert match[1] == 'https' + assert match[2] == valid_url_rest + + match = execute_in_page('returnval(proto_regex.exec(arguments[0]));', + '://bad-url.missing/protocol') + assert match is None + + # Test matching of http(s) URLs. + match = execute_in_page('returnval(http_regex.exec(arguments[0]));', + valid_url_rest) + assert match + assert match[1] == 'example.com' + assert match[2] == '/a/b' + assert match[3] == '?ver=1.2.3' + + match = execute_in_page('returnval(http_regex.exec(arguments[0]));', + 'another.example.com') + assert match + assert match[1] == 'another.example.com' + assert match[2] == '' + assert match[3] == '' + + match = execute_in_page('returnval(http_regex.exec(arguments[0]));', + '/bad/http/example') + assert match == None + + # Test matching of file URLs. + match = execute_in_page('returnval(file_regex.exec(arguments[0]));', + '/good/file/example') + assert match + assert match[1] == '/good/file/example' + + # Test matching of ftp URLs. + match = execute_in_page('returnval(ftp_regex.exec(arguments[0]));', + 'example.com/a/b#heading2') + assert match + assert match[1] is None + assert match[2] == 'example.com' + assert match[3] == '/a/b' + + match = execute_in_page('returnval(ftp_regex.exec(arguments[0]));', + 'some_user@localhost') + assert match + assert match[1] == 'some_user@' + assert match[2] == 'localhost' + assert match[3] == '' + + match = execute_in_page('returnval(ftp_regex.exec(arguments[0]));', + '@bad.url/') + assert match is None diff --git a/test/world_wide_library.py b/test/world_wide_library.py index fc9e095..860c987 100644 --- a/test/world_wide_library.py +++ b/test/world_wide_library.py @@ -1,15 +1,39 @@ -# Copyright (C) 2021 jahoti <jahoti@tilde.team> -# Licensing information is collated in the `copyright` file +# SPDX-License-Identifier: AGPL-3.0-or-later """ Our helpful little stand-in for the Internet """ +# This file is part of Haketilo. +# +# Copyright (C) 2021 jahoti <jahoti@tilde.team> +# Copyright (C) 2021 Wojtek Kosior <koszko@koszko.org> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this +# file's license. Although I request that you do not make use this code +# in a proprietary program, I am not going to enforce this in court. + +from .misc_constants import here + catalog = { - 'http://gotmyowndoma.in': (302, {'location': 'http://gotmyowndoma.in/index.html'}, b''), - 'http://gotmyowndoma.in/': (302, {'location': 'http://gotmyowndoma.in/index.html'}, b''), - 'http://gotmyowndoma.in/index.html': (200, {}, 'data/pages/gotmyowndomain.html'), - 'https://gotmyowndoma.in': (302, {'location': 'https://gotmyowndoma.in/index.html'}, b''), - 'https://gotmyowndoma.in/': (302, {'location': 'https://gotmyowndoma.in/index.html'}, b''), - 'https://gotmyowndoma.in/index.html': (200, {}, 'data/pages/gotmyowndomain_https.html') + 'http://gotmyowndoma.in': (302, {'location': 'http://gotmyowndoma.in/index.html'}, None), + 'http://gotmyowndoma.in/': (302, {'location': 'http://gotmyowndoma.in/index.html'}, None), + 'http://gotmyowndoma.in/index.html': (200, {}, here / 'data' / 'pages' / 'gotmyowndomain.html'), + 'https://gotmyowndoma.in': (302, {'location': 'https://gotmyowndoma.in/index.html'}, None), + 'https://gotmyowndoma.in/': (302, {'location': 'https://gotmyowndoma.in/index.html'}, None), + 'https://gotmyowndoma.in/index.html': (200, {}, here / 'data' / 'pages' / 'gotmyowndomain_https.html') } diff --git a/upload_amo.sh b/upload_amo.sh new file mode 100755 index 0000000..71e12ca --- /dev/null +++ b/upload_amo.sh @@ -0,0 +1,113 @@ +#!/bin/sh + +# This file is part of Haketilo +# +# Copyright (C) 2021, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +set -e + +. ./shell_utils.sh + +_PROG_NAME="$0" +OPERATION="$1" +API_KEY="$2" +SECRET="$3" +XPI_PATH="$4" + +base64url() { + printf %s "$1" | base64 -w 0 | tr '/+' '_-' | tr -d '=' +} + +sha256hmac() { + base64url "$(printf %s "$2" | openssl dgst -sha256 -hmac "$1" -binary -)" +} + +get_manifest_key() { + get_json_key "$1" "$(unzip -p "$2" manifest.json)" +} + +generate_jwt() { + local JWT_HEAD='{"alg":"HS256", "typ":"JWT"}' + local JWT_ID=$(dd if=/dev/random bs=21 count=1 2>/dev/null | base64) + local ISSUED_AT_TIME=$(date -u +%s) + local EXPIRATION_TIME=$((ISSUED_AT_TIME + 300)) + local JWT_PAYLOAD="$(cat <<EOF +{ + "iss": "$API_KEY", + "jti": "$JWT_ID", + "iat": $ISSUED_AT_TIME, + "exp": $EXPIRATION_TIME +} +EOF + )" + local JWT_MESSAGE=$(base64url "$JWT_HEAD").$(base64url "$JWT_PAYLOAD") + local JWT_SIGNATURE=$(sha256hmac "$SECRET" "$JWT_MESSAGE") + local JWT=$JWT_MESSAGE.$JWT_SIGNATURE + printf "Using JWT: $JWT\n" >&2 + printf $JWT +} + +get_extension_url() { + EXTENSION_ID="$(get_manifest_key id "$XPI_PATH")" + EXTENSION_VER="$(get_manifest_key version "$XPI_PATH")" + + if [ -z "$EXTENSION_ID" -o -z "$EXTENSION_VER" ]; then + printf "Couldn't extract extension id and version. Please check if %s contains proper manifest.json file.\n" \ + "$XPI_PATH" >&2 + exit 1 + fi + + printf 'https://addons.mozilla.org/api/v4/addons/%s/versions/%s/' \ + "$EXTENSION_ID" "$EXTENSION_VER" +} + +print_usage() { + printf 'Usage: %s upload|check|test API_KEY SECRET XPI_PATH\n' \ + "$_PROG_NAME" >&2 +} + +if [ $# != 4 ]; then + print_usage + exit 1 +fi + +unset RETURNED_DATA + +case "$OPERATION" in + test) + curl "https://addons.mozilla.org/api/v4/accounts/profile/" \ + -g -H "Authorization: JWT $(generate_jwt)" + printf '\n' + ;; + check) + RETURNED_DATA="$(curl $(get_extension_url) \ + -g -H "Authorization: JWT $(generate_jwt)")" + ;; + upload) + RETURNED_DATA="$(curl $(get_extension_url) \ + -g -XPUT --form "upload=@$XPI_PATH" \ + -H "Authorization: JWT $(generate_jwt)")" + ;; + *) + print_usage + exit 1 + ;; +esac + +if [ -n "$RETURNED_DATA" ]; then + printf "addons.mozilla.org says:\n%s\n" "$RETURNED_DATA" + DOWNLOAD_URL="$(get_json_key download_url "$RETURNED_DATA")" + if [ -n "$DOWNLOAD_URL" ]; then + printf "Downloading extension file from %s\n" "$DOWNLOAD_URL" + curl "$DOWNLOAD_URL" -g -H "Authorization: JWT $(generate_jwt)" -O + fi +fi diff --git a/write_makefile.sh b/write_makefile.sh new file mode 100755 index 0000000..4011fe8 --- /dev/null +++ b/write_makefile.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# This file is part of Haketilo +# +# Copyright (C) 2021, jahoti +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +if [ ! -e record.conf ]; then + printf "Record of configuration 'record.conf' does not exist.\n" >&2 + exit 1 +elif [ "$(head -n 1 record.conf | cut -c -9)x" != "srcdir = x" ]; then + printf "Record of configuration 'record.conf' is invalid.\n" >&2 + exit 2 +fi + +SRCDIR="$(head -n 1 record.conf | cut -c 10-)" +. "$SRCDIR"/shell_utils.sh +VERSION=$(get_json_key version "$(cat "$SRCDIR"/manifest.json)") + +sed '/^# Configuration goes here$/r record.conf +s|<<VPATH>>|'"$SRCDIR"'| +s/<<VERSION>>/'$VERSION/ < "$SRCDIR"/Makefile.in > Makefile |
