diff options
26 files changed, 838 insertions, 329 deletions
diff --git a/.gitmodules b/.gitmodules index edb76b9..3603e77 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "schemas"] - path = schemas + path = schemas/1.x + url = ../hydrilla-json-schemas/ +[submodule "hydrilla-json-schemas-2.x"] + path = schemas/2.x url = ../hydrilla-json-schemas/ diff --git a/background/CORS_bypass_server.js b/background/CORS_bypass_server.js index 9805d41..d15db54 100644 --- a/background/CORS_bypass_server.js +++ b/background/CORS_bypass_server.js @@ -2,7 +2,7 @@ * This file is part of Haketilo. * * Function: Allow other parts of the extension to bypass CORS by routing their - * request through this background script using one-off messages. + * requests through this background script using one-off messages. * * Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> * @@ -43,35 +43,40 @@ */ #FROM common/browser.js IMPORT browser +#FROM common/misc.js IMPORT uint8_to_hex, error_data_jsonifiable -async function get_prop(object, prop, result_object, call_prop=false) { - try { - result_object[prop] = call_prop ? (await object[prop]()) : object[prop]; - } catch(e) { - result_object[`error_${prop}`] = "" + e; - } +/* + * In this file we implement a fetch()-from-background-script service. Code in + * other parts of the extension shall call sendMessage() with arguments to + * fetch() and code here will call fetch() with those arguments and send back + * the response. + * + * We have to convert the Response from fetch() into a JSON-ifiable value and + * then convert it back (using Response() constructor) due (among others) the + * limitations of Chromium's messaging API: + * https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Chrome_incompatibilities#data_cloning_algorithm + * + * We also catch possible errors from fetch() (e.g. in case of an invalid URL) + * and send their data in JSON-ifiable form so that the error object can be + * later re-created. + */ + +/* Make it possible to serialize Response object. */ +async function response_data_jsonifiable(response) { + return { + status: response.status, + statusText: response.statusText, + headers: [...response.headers.entries()], + body: uint8_to_hex(new Uint8Array(await response.arrayBuffer())) + }; } -async function perform_download(fetch_data, respond_cb) { +async function perform_download(fetch_data) { try { - const response = await fetch(fetch_data.url); - const result = {}; - - for (const prop of (fetch_data.to_get || [])) - get_prop(response, prop, result); - - const to_call = (fetch_data.to_call || []); - const promises = []; - for (let i = 0; i < to_call.length; i++) { - const response_to_use = i === to_call.length - 1 ? - response : response.clone(); - promises.push(get_prop(response_to_use, to_call[i], result, true)); - } - - await Promise.all(promises); - return result; + const response = await fetch(fetch_data.url, fetch_data.init); + return response_data_jsonifiable(response); } catch(e) { - return {error: "" + e}; + return {error: error_data_jsonifiable(e)}; } } diff --git a/common/entities.js b/common/entities.js index 74cad20..41d6e3b 100644 --- a/common/entities.js +++ b/common/entities.js @@ -116,6 +116,28 @@ function* get_used_files(item) } #EXPORT get_used_files AS get_files +/* + * Regex to parse URIs like: + * https://hydrilla.koszko.org/schemas/api_mapping_description-2.schema.json + */ +const name_base_re = "(?<name_base>[^/]*)"; +const major_number_re = "(?<major>[1-9][0-9]*)"; +const minor_number_re = "(?:[1-9][0-9]*|0)"; +const numbers_rest_re = `(?:\\.${minor_number_re})*`; +const version_re = `(?<ver>${major_number_re}${numbers_rest_re})`; +const schema_name_re = `${name_base_re}-${version_re}\\.schema\\.json`; + +const haketilo_schema_name_regex = new RegExp(schema_name_re); +#EXPORT haketilo_schema_name_regex + +/* Extract the number that indicates entity's compatibility mode. */ +function get_schema_major_version(instance) { + const match = haketilo_schema_name_regex.exec(instance.$schema); + + return parseInt(match.groups.major); +} +#EXPORT get_schema_major_version + #IF NEVER /* diff --git a/common/jsonschema.js b/common/jsonschema.js index cde3fca..9c4a70c 100644 --- a/common/jsonschema.js +++ b/common/jsonschema.js @@ -57,6 +57,8 @@ #FROM common/jsonschema/scan.js IMPORT SchemaScanResult, scan +#FROM common/entities.js IMPORT haketilo_schema_name_regex + #EXPORT scan #EXPORT SchemaScanResult @@ -67,14 +69,32 @@ function validate(instance, schema, options) { #EXPORT validate const haketilo_schemas = [ -#INCLUDE schemas/api_query_result-1.0.1.schema.json + /* 1.x Hydrilla JSON schema series */ +#INCLUDE schemas/1.x/api_query_result-1.0.1.schema.json + , +#INCLUDE schemas/1.x/api_mapping_description-1.0.1.schema.json + , +#INCLUDE schemas/1.x/api_resource_description-1.0.1.schema.json + , +#INCLUDE schemas/1.x/common_definitions-1.0.1.schema.json , -#INCLUDE schemas/api_mapping_description-1.0.1.schema.json + /* 2.x Hydrilla JSON schema series */ +#INCLUDE schemas/2.x/api_query_result-2.schema.json , -#INCLUDE schemas/api_resource_description-1.0.1.schema.json +#INCLUDE schemas/2.x/api_mapping_description-2.schema.json , -#INCLUDE schemas/common_definitions-1.0.1.schema.json +#INCLUDE schemas/2.x/api_resource_description-2.schema.json + , +#INCLUDE schemas/2.x/common_definitions-2.schema.json ].reduce((ac, s) => Object.assign(ac, {[s.$id]: s}), {}); + +for (const [$id, schema] of [...Object.entries(haketilo_schemas)]) { + const match = haketilo_schema_name_regex.exec($id); + const schema_name = + `${match.groups.name_base}-${match.groups.major}.schema.json`; + haketilo_schemas[schema_name] = schema; +} + #EXPORT haketilo_schemas const haketilo_validator = new Validator(); diff --git a/common/misc.js b/common/misc.js index e609fe0..8096f27 100644 --- a/common/misc.js +++ b/common/misc.js @@ -42,9 +42,13 @@ * proprietary program, I am not going to enforce this in court. */ -/* uint8_to_hex is a separate function used in cryptographic functions. */ +/* + * uint8_to_hex is a separate function used in cryptographic functions and when + * dealing with binary data. + */ const uint8_to_hex = array => [...array].map(b => ("0" + b.toString(16)).slice(-2)).join(""); +#EXPORT uint8_to_hex /* * Asynchronously compute hex string representation of a sha256 digest of a @@ -61,8 +65,7 @@ async function sha256_async(string) { * Generate a unique value that can be computed synchronously and is impossible * to guess for a malicious website. */ -function gen_nonce(length=16) -{ +function gen_nonce(length=16) { const random_data = new Uint8Array(length); crypto.getRandomValues(random_data); return uint8_to_hex(random_data); @@ -71,16 +74,10 @@ function gen_nonce(length=16) /* Check if some HTTP header might define CSP rules. */ const csp_header_regex = - /^\s*(content-security-policy|x-webkit-csp|x-content-security-policy)/i; + /^\s*(content-security-policy|x-webkit-csp|x-content-security-policy)\s*$/i; #EXPORT csp_header_regex /* - * Print item together with type, e.g. - * nice_name("s", "hello") → "hello (script)" - */ -#EXPORT (prefix, name) => `${name} (${TYPE_NAME[prefix]})` AS nice_name - -/* * Check if url corresponds to a browser's special page (or a directory index in * case of `file://' protocol). */ @@ -90,3 +87,13 @@ const priv_reg = /^moz-extension:\/\/|^about:|^view-source:|^file:\/\/[^?#]*\/([ const priv_reg = /^chrome(-extension)?:\/\/|^about:|^view-source:|^file:\/\/[^?#]*\/([?#]|$)/; #ENDIF #EXPORT url => priv_reg.test(url) AS is_privileged_url + +/* Make it possible to serialize en Error object. */ +function error_data_jsonifiable(error) { + const jsonifiable = {}; + for (const property of ["name", "message", "fileName", "lineNumber"]) + jsonifiable[property] = error[property]; + + return jsonifiable; +} +#EXPORT error_data_jsonifiable diff --git a/common/policy.js b/common/policy.js index e14d8cd..6bcb54b 100644 --- a/common/policy.js +++ b/common/policy.js @@ -49,16 +49,15 @@ * CSP rule that either blocks all scripts or only allows scripts with specified * nonce attached. */ -function make_csp(nonce) -{ - const rule = nonce ? `nonce-${nonce}` : "none"; +function make_csp(nonce) { + const rule = nonce ? `'nonce-${nonce}'` : "'none'"; const csp_list = [ - ["prefetch-src", "none"], - ["script-src-attr", "none"], - ["script-src", rule], + ["prefetch-src", "'none'"], + ["script-src-attr", "'none'"], + ["script-src", rule, "'unsafe-eval'"], ["script-src-elem", rule] ]; - return csp_list.map(([a, b]) => `${a} '${b}';`).join(" "); + return csp_list.map(words => `${words.join(" ")};`).join(" "); } function decide_policy(patterns_tree, url, default_allow, secret) @@ -113,3 +112,33 @@ function decide_policy(patterns_tree, url, default_allow, secret) #EXPORT decide_policy #EXPORT () => ({allow: false, csp: make_csp()}) AS fallback_policy + +#IF NEVER + +/* + * Note: the functions below were overeagerly written and are not used now but + * might prove useful to once we add more functionalities and are hence kept... + */ + +function relaxed_csp_eval(csp) { + const new_csp_list = []; + + for (const directive of csp.split(";")) { + const directive_words = directive.trim().split(" "); + if (directive_words[0] === "script-src") + directive_words.push("'unsafe-eval'"); + + new_csp_list.push(directive_words); + } + + new_policy.csp = new_csp_list.map(d => `${d.join(" ")}';`).join(" "); +} + +function relax_policy_eval(policy) { + const new_policy = Object.assign({}, policy); + + return Object.assign(new_policy, {csp: relaxed_csp_eval(policy.csp)}); +} +#EXPORT relax_policy_eval + +#ENDIF diff --git a/content/content.js b/content/content.js index ef2ee39..c492d53 100644 --- a/content/content.js +++ b/content/content.js @@ -42,6 +42,7 @@ */ #IMPORT content/repo_query_cacher.js +#IMPORT content/haketilo_apis.js #FROM common/browser.js IMPORT browser #FROM common/misc.js IMPORT is_privileged_url @@ -132,7 +133,14 @@ async function main() { resolve_page_info(Object.assign(page_info, script_response)); return; } else { - for (const script_contents of script_response.files) { + haketilo_apis.start(); + + const version = browser.runtime.getManifest().version; + const scripts = [ + `window.haketilo_version = ${JSON.stringify(version)};`, + ...script_response.files + ]; + for (const script_contents of scripts) { const html_ns = "http://www.w3.org/1999/xhtml"; const script = document.createElementNS(html_ns, "script"); diff --git a/content/haketilo_apis.js b/content/haketilo_apis.js new file mode 100644 index 0000000..ccfec37 --- /dev/null +++ b/content/haketilo_apis.js @@ -0,0 +1,77 @@ +/** + * This file is part of Haketilo. + * + * Function: Expose some special functionalities to injected scripts using + * CustomEvent's to communicate with them. + * + * Copyright (C) 2022 Wojtek Kosior + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * As additional permission under GNU GPL version 3 section 7, you + * may distribute forms of that code without the copy of the GNU + * GPL normally required by section 4, provided you include this + * license notice and, in case of non-source distribution, a URL + * through which recipients can access the Corresponding Source. + * If you modify file(s) with this exception, you may extend this + * exception to your version of the file(s), but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * As a special exception to the GPL, any HTML file which merely + * makes function calls to this code, and for that purpose + * includes it by reference shall be deemed a separate work for + * copyright law purposes. If you modify this code, you may extend + * this exception to your version of the code, but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + * + * I, Wojtek Kosior, thereby promise not to sue for violation of this file's + * license. Although I request that you do not make use of this code in a + * proprietary program, I am not going to enforce this in court. + */ + +#FROM common/browser.js IMPORT browser +#FROM common/misc.js IMPORT error_data_jsonifiable + +async function on_CORS_bypass(event) { + const name = "haketilo_CORS_bypass"; + + if (typeof event.detail !== "object" || + event.detail === null || + typeof event.detail.id !== "string" || + typeof event.detail.data !== "string") { + console.error(`Haketilo: Invalid detail for ${name}:`, + event.detail); + return; + } + + try { + const data = JSON.parse(event.detail.data); + var result = await browser.runtime.sendMessage(["CORS_bypass", data]); + if (result === undefined) + throw new Error("Couldn't communicate with Haketilo background script."); + } catch(e) { + var result = {error: error_data_jsonifiable(e)}; + } + + const response_name = `${name}-${event.detail.id}`; + const detail = JSON.stringify(result); + window.dispatchEvent(new CustomEvent(response_name, {detail})); +} + +function start() { + window.addEventListener("haketilo_CORS_bypass", on_CORS_bypass); +} +#EXPORT start diff --git a/content/repo_query_cacher.js b/content/repo_query_cacher.js index 3f62be7..be97c39 100644 --- a/content/repo_query_cacher.js +++ b/content/repo_query_cacher.js @@ -43,10 +43,12 @@ */ #FROM common/browser.js IMPORT browser +#FROM common/misc.js IMPORT error_data_jsonifiable /* * Map URLs to objects containing parsed responses, error info or promises - * resolving to those. + * resolving to those. The use of promises helps us prevent multiple requests + * for the same resource from starting concurrently. */ const cache = new Map(); @@ -58,12 +60,11 @@ async function perform_download(url) { cache.set(url, new Promise(cb => resolve_cb = cb)); try { - const opts = {url, to_get: ["ok", "status"], to_call: ["json"]}; - var result = await browser.runtime.sendMessage(["CORS_bypass", opts]); + var result = await browser.runtime.sendMessage(["CORS_bypass", {url}]); if (result === undefined) - result = {error: "Couldn't communicate with background script."}; + throw new Error("Couldn't communicate with background script."); } catch(e) { - var result = {error: e + ""}; + return {error: error_data_jsonifiable(e)}; } cache.set(url, result); diff --git a/html/install.js b/html/install.js index fddee5d..82df661 100644 --- a/html/install.js +++ b/html/install.js @@ -50,6 +50,9 @@ #FROM common/entities.js IMPORT item_id_string, version_string, get_files #FROM common/misc.js IMPORT sha256_async AS compute_sha256 #FROM common/jsonschema.js IMPORT haketilo_validator, haketilo_schemas +#FROM common/entities.js IMPORT haketilo_schema_name_regex + +#FROM html/repo_query_cacher_client.js IMPORT indirect_fetch const coll = new Intl.Collator(); @@ -104,6 +107,9 @@ async function init_work() { }; work.err = function (error, user_message) { + if (!this.is_ok) + return; + if (error) console.error("Haketilo:", error); work.is_ok = false; @@ -171,54 +177,71 @@ function InstallView(tab_id, on_view_show, on_view_hide) { const url = ver ? `${this.repo_url}${item_type}/${id}/${ver.join(".")}` : `${this.repo_url}${item_type}/${id}.json`; - const response = - await browser.tabs.sendMessage(tab_id, ["repo_query", url]); - if (!work.is_ok) - return; - if ("error" in response) { - return work.err(response.error, - "Failure to communicate with repository :("); + + try { + var response = await indirect_fetch(tab_id, url); + } catch(e) { + return work.err(e, "Failure to communicate with repository :("); } + if (!work.is_ok) + return; + if (!response.ok) { return work.err(null, `Repository sent HTTP code ${response.status} :(`); } - if ("error_json" in response) { - return work.err(response.error_json, - "Repository's response is not valid JSON :("); + try { + var json = await response.json(); + } catch(e) { + return work.err(e, "Repository's response is not valid JSON :("); } + if (!work.is_ok) + return; + const captype = item_type[0].toUpperCase() + item_type.substring(1); - const $id = - `https://hydrilla.koszko.org/schemas/api_${item_type}_description-1.0.1.schema.json`; - const schema = haketilo_schemas[$id]; - const result = haketilo_validator.validate(response.json, schema); - if (result.errors.length > 0) { - const reg = new RegExp(schema.allOf[2].properties.$schema.pattern); - if (response.json.$schema && !reg.test(response.json.$schema)) { + const nonconforming_format_error_msg = + `${captype} ${item_id_string(id, ver)} was served using a nonconforming response format.`; + + try { + const match = haketilo_schema_name_regex.exec(json.$schema); + var major_schema_version = match.groups.major; + + if (!["1", "2"].includes(major_schema_version)) { const msg = `${captype} ${item_id_string(id, ver)} was served using unsupported Hydrilla API version. You might need to update Haketilo.`; - return work.err(result.errors, msg); + return work.err(null, msg); } - - const msg = `${captype} ${item_id_string(id, ver)} was served using a nonconforming response format.`; - return work.err(result.errors, msg); + } catch(e) { + return work.err(e, nonconforming_format_error_msg); } - const scripts = item_type === "resource" && response.json.scripts; - const files = response.json.source_copyright.concat(scripts || []); + const schema_name = `api_${item_type}_description-${major_schema_version}.schema.json`; + + const schema = haketilo_schemas[schema_name]; + const result = haketilo_validator.validate(json, schema); + if (result.errors.length > 0) + return work.err(result.errors, nonconforming_format_error_msg); + + const scripts = item_type === "resource" && json.scripts; + const files = json.source_copyright.concat(scripts || []); if (item_type === "mapping") { - for (const res_ref of Object.values(response.json.payloads || {})) + for (const res_ref of Object.values(json.payloads || {})) process_item(work, "resource", res_ref.identifier); } else { - for (const res_ref of (response.json.dependencies || [])) + for (const res_ref of (json.dependencies || [])) process_item(work, "resource", res_ref.identifier); } + if (major_schema_version >= 2) { + for (const map_ref of (json.required_mappings || [])) + process_item(work, "mapping", map_ref.identifier); + } + /* * At this point we already have JSON definition of the item and we * triggered processing of its dependencies. We now have to verify if @@ -234,8 +257,8 @@ function InstallView(tab_id, on_view_show, on_view_hide) { const msg = "Error accessing Haketilo's internal database :("; return work.err(e, msg); } - if (!db_def || db_def.version < response.json.version) - work.result.push({def: response.json, db_def}); + if (!db_def || db_def.version < json.version) + work.result.push({def: json, db_def}); if (--work.waiting === 0) work.resolve_cb(work.result); @@ -320,14 +343,9 @@ function InstallView(tab_id, on_view_show, on_view_hide) { return work.err(null, msg); } - try { - var text = await response.text(); - if (!work.is_ok) - return; - } catch(e) { - const msg = "Repository's response is not valid text :("; - return work.err(e, msg); - } + const text = await response.text(); + if (!work.is_ok) + return; const digest = await compute_sha256(text); if (!work.is_ok) diff --git a/html/repo_query.js b/html/repo_query.js index 601a0aa..97f60ac 100644 --- a/html/repo_query.js +++ b/html/repo_query.js @@ -49,6 +49,8 @@ #FROM html/install.js IMPORT InstallView #FROM common/jsonschema.js IMPORT haketilo_validator, haketilo_schemas +#FROM html/repo_query_cacher_client.js IMPORT indirect_fetch + const coll = new Intl.Collator(); function ResultEntry(repo_entry, mapping_ref) { @@ -76,36 +78,42 @@ function RepoEntry(query_view, repo_url) { this.repo_url_label.innerText = repo_url; - const query_results = async () => { - const msg = [ - "repo_query", - `${repo_url}query?url=${encodeURIComponent(query_view.url)}` - ]; - const response = await browser.tabs.sendMessage(query_view.tab_id, msg); + const encoded_queried_url = encodeURIComponent(query_view.url); + const query_url = `${repo_url}query?url=${encoded_queried_url}`; - if ("error" in response) + const query_results = async () => { + try { + var response = await indirect_fetch(query_view.tab_id, query_url); + } catch(e) { + console.error("Haketilo:", e); throw "Failure to communicate with repository :("; + } if (!response.ok) throw `Repository sent HTTP code ${response.status} :(`; - if ("error_json" in response) + + try { + var json = await response.json(); + } catch(e) { + console.error("Haketilo:", e); throw "Repository's response is not valid JSON :("; + } const $id = `https://hydrilla.koszko.org/schemas/api_query_result-1.0.1.schema.json`; const schema = haketilo_schemas[$id]; - const result = haketilo_validator.validate(response.json, schema); + const result = haketilo_validator.validate(json, schema); if (result.errors.length > 0) { console.error("Haketilo:", result.errors); const reg = new RegExp(schema.properties.$schema.pattern); - if (response.json.$schema && !reg.test(response.json.$schema)) + if (json.$schema && !reg.test(json.$schema)) throw "Results were served using unsupported Hydrilla API version. You might need to update Haketilo."; throw "Results were served using a nonconforming response format."; } - return response.json.mappings; + return json.mappings; } const populate_results = async () => { diff --git a/html/repo_query_cacher_client.js b/html/repo_query_cacher_client.js new file mode 100644 index 0000000..b7f49e9 --- /dev/null +++ b/html/repo_query_cacher_client.js @@ -0,0 +1,85 @@ +/** + * This file is part of Haketilo. + * + * Function: Making requests to remote repositories through a response cache in + * operating in the content script of a browser tab. + * + * Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * As additional permission under GNU GPL version 3 section 7, you + * may distribute forms of that code without the copy of the GNU + * GPL normally required by section 4, provided you include this + * license notice and, in case of non-source distribution, a URL + * through which recipients can access the Corresponding Source. + * If you modify file(s) with this exception, you may extend this + * exception to your version of the file(s), but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * As a special exception to the GPL, any HTML file which merely + * makes function calls to this code, and for that purpose + * includes it by reference shall be deemed a separate work for + * copyright law purposes. If you modify this code, you may extend + * this exception to your version of the code, but you are not + * obligated to do so. If you do not wish to do so, delete this + * exception statement from your version. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + * + * I, Wojtek Kosior, thereby promise not to sue for violation of this file's + * license. Although I request that you do not make use of this code in a + * proprietary program, I am not going to enforce this in court. + */ + +#FROM common/browser.js IMPORT browser + +/* + * This is a complementary function to error_data_jsonifiable() in + * common/misc.js. + */ +function error_from_jsonifiable_data(jsonifiable) { + const arg_props = ["message", "fileName", "lineNumber"]; + return new window[jsonifiable.name](...arg_props.map(p => jsonifiable[p])); +} + +/* + * Make it possible to recover a Response object. This is a complementary + * function to response_data_jsonifiable() in background/CORS_bypass_server.js. + */ +function response_from_jsonifiable_data(jsonifiable) { + const body = jsonifiable.body, body_len = body.length / 2; + const body_buf = new Uint8Array(body_len); + + for (let i = 0; i < body_len; i++) + body_buf[i] = parseInt(`0x${body.substring(i * 2, i * 2 + 2)}`); + + const init = { + status: jsonifiable.status, + statusText: jsonifiable.statusText, + headers: new Headers(jsonifiable.headers) + }; + + return new Response(body_buf, init); +} + +async function indirect_fetch(tab_id, url) { + const msg = ["repo_query", url]; + const jsonifiable = await browser.tabs.sendMessage(tab_id, msg); + + if ("error" in jsonifiable) + throw error_from_jsonifiable_data(jsonifiable); + + return response_from_jsonifiable_data(jsonifiable); +} +#EXPORT indirect_fetch diff --git a/schemas b/schemas/1.x -Subproject 09634f3446866f712a022327683b1149d8f46bf +Subproject 09634f3446866f712a022327683b1149d8f46bf diff --git a/schemas/2.x b/schemas/2.x new file mode 160000 +Subproject 7206db45f277c10c34d1b7ed9bd35343ac742d3 diff --git a/test/haketilo_test/unit/test_CORS_bypass_server.py b/test/haketilo_test/unit/test_CORS_bypass_server.py index 45e4ebb..45f06a9 100644 --- a/test/haketilo_test/unit/test_CORS_bypass_server.py +++ b/test/haketilo_test/unit/test_CORS_bypass_server.py @@ -24,29 +24,28 @@ from selenium.webdriver.support.ui import WebDriverWait from ..script_loader import load_script from ..world_wide_library import some_data -urls = { - 'resource': 'https://anotherdoma.in/resource/blocked/by/CORS.json', - 'nonexistent': 'https://nxdoma.in/resource.json', - 'invalid': 'w3csucks://invalid.url/' +datas = { + 'resource': 'https://anotherdoma.in/resource/blocked/by/CORS.json', + 'nonexistent': 'https://nxdoma.in/resource.json', + 'invalid': 'w3csucks://invalid.url/', + 'redirected_ok': 'https://site.with.scripts.block.ed', + 'redirected_err': 'https://site.with.scripts.block.ed' } +for name, url in [*datas.items()]: + datas[name] = {'url': url} + +datas['redirected_ok']['init'] = {'redirect': 'follow'} +datas['redirected_err']['init'] = {'redirect': 'error'} + content_script = '''\ -const urls = %s; - -function fetch_data(url) { - return { - url, - to_get: ["ok", "status"], - to_call: ["text", "json"] - }; -} +const datas = %s; async function fetch_resources() { const results = {}; const promises = []; - for (const [name, url] of Object.entries(urls)) { - const sending = browser.runtime.sendMessage(["CORS_bypass", - fetch_data(url)]); + for (const [name, data] of Object.entries(datas)) { + const sending = browser.runtime.sendMessage(["CORS_bypass", data]); promises.push(sending.then(response => results[name] = response)); } @@ -58,7 +57,7 @@ async function fetch_resources() { fetch_resources(); ''' -content_script = content_script % json.dumps(urls); +content_script = content_script % json.dumps(datas); @pytest.mark.ext_data({ 'content_script': content_script, @@ -77,33 +76,41 @@ def test_CORS_bypass_server(driver, execute_in_page): ''' const result = {}; let promises = []; - for (const [name, url] of Object.entries(arguments[0])) { + for (const [name, data] of Object.entries(arguments[0])) { const [ok_cb, err_cb] = ["ok", "err"].map(status => () => result[name] = status); - promises.push(fetch(url).then(ok_cb, err_cb)); + promises.push(fetch(data.url).then(ok_cb, err_cb)); } // Make the promises non-failing. promises = promises.map(p => new Promise(cb => p.then(cb, cb))); returnval(Promise.all(promises).then(() => result)); ''', - {**urls, 'sameorigin': './nonexistent_resource'}) + {**datas, 'sameorigin': './nonexistent_resource'}) - assert results == dict([*[(k, 'err') for k in urls.keys()], + assert results == dict([*[(k, 'err') for k in datas.keys()], ('sameorigin', 'ok')]) done = lambda d: d.execute_script('return window.haketilo_fetch_results;') results = WebDriverWait(driver, 10).until(done) assert set(results['invalid'].keys()) == {'error'} + assert results['invalid']['error']['fileName'].endswith('background.js') + assert type(results['invalid']['error']['lineNumber']) is int + assert type(results['invalid']['error']['message']) is str + assert results['invalid']['error']['name'] == 'TypeError' - assert set(results['nonexistent'].keys()) == \ - {'ok', 'status', 'text', 'error_json'} - assert results['nonexistent']['ok'] == False assert results['nonexistent']['status'] == 404 - assert results['nonexistent']['text'] == 'Handler for this URL not found.' + assert results['nonexistent']['statusText'] == 'Not Found' + assert any([name.lower() == 'content-length' + for name, value in results['nonexistent']['headers']]) + assert bytes.fromhex(results['nonexistent']['body']) == \ + b'Handler for this URL not found.' - assert set(results['resource'].keys()) == {'ok', 'status', 'text', 'json'} - assert results['resource']['ok'] == True assert results['resource']['status'] == 200 - assert results['resource']['text'] == some_data - assert results['resource']['json'] == json.loads(some_data) + assert results['resource']['statusText'] == 'OK' + assert any([name.lower() == 'content-length' + for name, value in results['resource']['headers']]) + assert bytes.fromhex(results['resource']['body']) == b'{"some": "data"}' + + assert results['redirected_ok']['status'] == 200 + assert results['redirected_err']['error']['name'] == 'TypeError' diff --git a/test/haketilo_test/unit/test_content.py b/test/haketilo_test/unit/test_content.py index 8220160..98ea930 100644 --- a/test/haketilo_test/unit/test_content.py +++ b/test/haketilo_test/unit/test_content.py @@ -88,6 +88,7 @@ content_script = \ } repo_query_cacher.start = () => data_set("cacher_started", true); + haketilo_apis.start = () => data_set("apis_started", true); enforce_blocking = policy => data_set("enforcing", policy); @@ -118,7 +119,7 @@ content_script = \ @pytest.mark.ext_data({'content_script': content_script}) @pytest.mark.usefixtures('webextension') -@pytest.mark.parametrize('target1', ['dynamic_before'])#, 'dynamic_after']) +@pytest.mark.parametrize('target1', ['dynamic_before', 'dynamic_after']) @pytest.mark.parametrize('target2', [ 'scripts_blocked', 'payload_error', @@ -144,6 +145,7 @@ def test_content_unprivileged_page(driver, execute_in_page, target1, target2): assert data['bad_request_returned'] == False assert data['cacher_started'] == True + assert data.get('apis_started', False) == (target2 == 'payload_ok') for obj in (data['good_request_result'], data['enforcing']): assert obj['allow'] == False @@ -162,9 +164,13 @@ def test_content_unprivileged_page(driver, execute_in_page, target1, target2): def vars_made_by_payload(driver): vars_values = driver.execute_script( - 'return [1, 2].map(n => window[`hak_injected_${n}`]);' - ) - if vars_values != [None, None]: + ''' + return [ + ...[1, 2].map(n => window[`hak_injected_${n}`]), + window.haketilo_version + ]; + ''') + if vars_values != [None, None, None]: return vars_values if target2 == 'payload_error': @@ -174,7 +180,8 @@ def test_content_unprivileged_page(driver, execute_in_page, target1, target2): } elif target2 == 'payload_ok': vars_values = WebDriverWait(driver, 10).until(vars_made_by_payload) - assert vars_values == [1, 2] + assert vars_values[:2] == [1, 2] + assert type(vars_values[2]) == str @pytest.mark.ext_data({'content_script': content_script}) @pytest.mark.usefixtures('webextension') diff --git a/test/haketilo_test/unit/test_haketilo_apis.py b/test/haketilo_test/unit/test_haketilo_apis.py new file mode 100644 index 0000000..af7906d --- /dev/null +++ b/test/haketilo_test/unit/test_haketilo_apis.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - exposing some special functionalities to injected scripts +""" + +# This file is part of Haketilo +# +# Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +import json +from selenium.webdriver.support.ui import WebDriverWait + +from ..script_loader import load_script +from ..world_wide_library import some_data + +def content_script(): + return load_script('content/haketilo_apis.js') + ';\nstart();' + +def background_script(): + return load_script('background/CORS_bypass_server.js') + ';\nstart();' + +resource_url = 'https://anotherdoma.in/resource/blocked/by/CORS.json' + +@pytest.mark.ext_data({ + 'content_script': content_script, + 'background_script': background_script +}) +@pytest.mark.usefixtures('webextension') +def test_haketilo_apis_CORS_bypass(driver): + """ + Verify injected scripts will be able to bypass CORS with the help of + Haketilo API. + """ + driver.get('https://gotmyowndoma.in/') + + # First, verify that it is impossible to normally fetch the resource. + with pytest.raises(Exception, match='NetworkError'): + driver.execute_script('return fetch(arguments[0]);', resource_url) + + # First, verify that it is possible to fetch the resource using API. + response = driver.execute_script( + ''' + const fetch_arg = { + url: arguments[0], + init: {}, + verify_that_nonstandard_properties_are_ignored: ":)" + }; + + const detail = { + data: JSON.stringify(fetch_arg), + id: "abcdef", + nonstandard_properties_verify_that_ignored_are: ":o" + }; + + let cb, done = new Promise(_cb => cb = _cb); + window.addEventListener("haketilo_CORS_bypass-abcdef", + e => cb(JSON.parse(e.detail))); + window.dispatchEvent(new CustomEvent("haketilo_CORS_bypass", {detail})); + + return done; + ''', + resource_url) + + assert response['body'] == some_data.encode().hex() + assert response['status'] == 200 + assert type(response['headers']) is list + +@pytest.mark.ext_data({ + 'content_script': content_script, + 'background_script': background_script +}) +@pytest.mark.usefixtures('webextension') +@pytest.mark.parametrize('error', [ + 'bad url', + 'no_url', + 'non_string_url', + 'non_object_init', + 'non_object_detail', + 'non_string_id', + 'non_string_data' +]) +def test_haketilo_apis_CORS_bypass_errors(driver, error): + """ + Verify errors are returned properly by CORS_bypass API. + """ + data = { + 'bad_url': {'url': 'muahahahaha', 'init': {}}, + 'no_url': {'init': {}}, + 'non_string_url': {'url': {}, 'init': {}}, + 'non_object_init': {'url': {}, 'init': ":d"}, + }.get(error, {'url': resource_url, 'init': {}}) + + detail = { + 'non_object_detail': '!!!', + 'non_string_id': {'data': json.dumps(data), 'id': None}, + 'non_string_data': {'data': data, 'id': 'abcdef'} + }.get(error, {'data': json.dumps(data), 'id': 'abcdef'}) + + driver.get('https://gotmyowndoma.in/') + + result = driver.execute_script( + ''' + let cb, done = new Promise(_cb => cb = _cb); + window.addEventListener("haketilo_CORS_bypass-abcdef", + e => cb(JSON.parse(e.detail))); + window.dispatchEvent(new CustomEvent("haketilo_CORS_bypass", + {detail: arguments[0]})); + setTimeout(() => cb("timeout"), 5000); + + return done; + ''', + detail) + + if error in {'bad_url', 'no_url', 'non_string_url', 'non_object_init'}: + assert result['error']['name'] == 'TypeError' + + if error in {'non_object_detail', 'non_string_id', 'non_string_data'}: + assert result == 'timeout' diff --git a/test/haketilo_test/unit/test_install.py b/test/haketilo_test/unit/test_install.py index 1e2063c..b1321ff 100644 --- a/test/haketilo_test/unit/test_install.py +++ b/test/haketilo_test/unit/test_install.py @@ -26,7 +26,7 @@ from ..script_loader import load_script from .utils import * def setup_view(driver, execute_in_page): - mock_cacher(execute_in_page) + execute_in_page(mock_cacher_code) execute_in_page(load_script('html/install.js')) container_ids, containers_objects = execute_in_page( @@ -57,8 +57,38 @@ install_ext_data = { @pytest.mark.ext_data(install_ext_data) @pytest.mark.usefixtures('webextension') -@pytest.mark.parametrize('complex_variant', [False, True]) -def test_install_normal_usage(driver, execute_in_page, complex_variant): +@pytest.mark.parametrize('variant', [{ + # The resource/mapping others depend on. + 'root_resource_id': f'resource-abcd-defg-ghij', + 'root_mapping_id': f'mapping-abcd-defg-ghij', + # Those ids are used to check the alphabetical ordering. + 'item_ids': [f'resource-{letters}' for letters in ( + 'a', 'abcd', 'abcd-defg-ghij', 'b', 'c', + 'd', 'defg', 'e', 'f', + 'g', 'ghij', 'h', 'i', 'j' + )], + 'files_count': 9 +}, { + 'root_resource_id': 'resource-a', + 'root_mapping_id': 'mapping-a', + 'item_ids': ['resource-a'], + 'files_count': 0 +}, { + 'root_resource_id': 'resource-a-w-required-mapping-v1', + 'root_mapping_id': 'mapping-a-w-required-mapping-v1', + 'item_ids': ['resource-a-w-required-mapping-v1'], + 'files_count': 1 +}, { + 'root_resource_id': 'resource-a-w-required-mapping-v2', + 'root_mapping_id': 'mapping-a-w-required-mapping-v2', + 'item_ids': [ + 'mapping-a', + 'resource-a', + 'resource-a-w-required-mapping-v2' + ], + 'files_count': 1 +}]) +def test_install_normal_usage(driver, execute_in_page, variant): """ Test of the normal package installation procedure with one mapping and, depending on parameter, one or many resources. @@ -67,41 +97,27 @@ def test_install_normal_usage(driver, execute_in_page, complex_variant): assert execute_in_page('returnval(shw());') == [[], False] - if complex_variant: - # The resource/mapping others depend on. - root_id = 'abcd-defg-ghij' - root_resource_id = f'resource-{root_id}' - root_mapping_id = f'mapping-{root_id}' - # Those ids are used to check the alphabetical ordering. - resource_ids = [f'resource-{letters}' for letters in ( - 'a', 'abcd', root_id, 'b', 'c', - 'd', 'defg', 'e', 'f', - 'g', 'ghij', 'h', 'i', 'j' - )] - files_count = 9 - else: - root_resource_id = f'resource-a' - root_mapping_id = f'mapping-a' - resource_ids = [root_resource_id] - files_count = 0 - # Preview the installation of a resource, show resource's details, close # the details and cancel installation. execute_in_page('returnval(install_view.show(...arguments));', - 'https://hydril.la/', 'resource', root_resource_id) + 'https://hydril.la/', 'resource', + variant['root_resource_id']) assert execute_in_page('returnval(shw());') == [['show'], True] - assert f'{root_resource_id}-2021.11.11-1'\ + assert f'{variant["root_resource_id"]}-2021.11.11-1'\ in containers['install_preview'].text assert_container_displayed('install_preview') entries = execute_in_page('returnval(ets().map(e => e.main_li.innerText));') - assert len(entries) == len(resource_ids) + assert len(entries) == len(variant['item_ids']) + resource_idx = variant['item_ids'].index(variant['root_resource_id']) # Verify alphabetical ordering. - assert all([id in text for id, text in zip(resource_ids, entries)]) + assert all([id in text for id, text in + zip(variant['item_ids'], entries)]) - assert not execute_in_page('returnval(ets()[0].old_ver);').is_displayed() - execute_in_page('returnval(ets()[0].details_but);').click() + assert not execute_in_page(f'returnval(ets()[{resource_idx}].old_ver);')\ + .is_displayed() + execute_in_page(f'returnval(ets()[{resource_idx}].details_but);').click() assert 'resource-a' in containers['resource_preview_container'].text assert_container_displayed('resource_preview_container') @@ -116,20 +132,24 @@ def test_install_normal_usage(driver, execute_in_page, complex_variant): # details, close the details and commit the installation. execute_in_page('returnval(install_view.show(...arguments));', 'https://hydril.la/', 'mapping', - root_mapping_id, [2022, 5, 10]) + variant['root_mapping_id'], [2022, 5, 10]) assert execute_in_page('returnval(shw(2));') == [['show'], True] assert_container_displayed('install_preview') entries = execute_in_page('returnval(ets().map(e => e.main_li.innerText));') - assert len(entries) == len(resource_ids) + 1 - assert f'{root_mapping_id}-2022.5.10' in entries[0] + assert len(entries) == len(variant['item_ids']) + 1 + + all_item_ids = sorted([*variant['item_ids'], variant['root_mapping_id']]) + mapping_idx = all_item_ids.index(variant["root_mapping_id"]) # Verify alphabetical ordering. - assert all([id in text for id, text in zip(resource_ids, entries[1:])]) + assert all([id in text for id, text in zip(all_item_ids, entries)]) - assert not execute_in_page('returnval(ets()[0].old_ver);').is_displayed() - execute_in_page('returnval(ets()[0].details_but);').click() - assert root_mapping_id in containers['mapping_preview_container'].text + assert not execute_in_page(f'returnval(ets()[{mapping_idx}].old_ver);')\ + .is_displayed() + execute_in_page(f'returnval(ets()[{mapping_idx}].details_but);').click() + assert variant['root_mapping_id'] in \ + containers['mapping_preview_container'].text assert_container_displayed('mapping_preview_container') execute_in_page('returnval(install_view.mapping_back_but);').click() @@ -145,16 +165,20 @@ def test_install_normal_usage(driver, execute_in_page, complex_variant): # Verify the install db_contents = get_db_contents(execute_in_page) - for item_type, ids in \ - [('mapping', {root_mapping_id}), ('resource', set(resource_ids))]: + all_map_ids = {id for id in all_item_ids if id.startswith('mapping')} + all_res_ids = {id for id in all_item_ids if id.startswith('resource')} + for item_type, ids in [ + ('mapping', all_map_ids), + ('resource', all_res_ids) + ]: assert set([it['identifier'] for it in db_contents[item_type]]) == ids - assert all([len(db_contents[store]) == files_count + assert all([len(db_contents[store]) == variant['files_count'] for store in ('file', 'file_uses')]) # Update the installed mapping to a newer version. execute_in_page('returnval(install_view.show(...arguments));', - 'https://hydril.la/', 'mapping', root_mapping_id) + 'https://hydril.la/', 'mapping', variant['root_mapping_id']) assert execute_in_page('returnval(shw(4));') == [['show'], True] # resources are already in the newest versions, hence they should not appear # in the install preview list. @@ -171,12 +195,19 @@ def test_install_normal_usage(driver, execute_in_page, complex_variant): # Verify the newer version install. old_db_contents, db_contents = db_contents, get_db_contents(execute_in_page) - old_db_contents['mapping'][0]['version'][-1] += 1 - assert db_contents['mapping'] == old_db_contents['mapping'] + + old_root_mapping = [m for m in old_db_contents['mapping'] + if m['identifier'] == variant['root_mapping_id']][0] + old_root_mapping['version'][-1] += 1 + + new_root_mapping = [m for m in db_contents['mapping'] + if m['identifier'] == variant['root_mapping_id']][0] + + assert old_root_mapping == new_root_mapping # All items are up to date - verify dialog is instead shown in this case. execute_in_page('install_view.show(...arguments);', - 'https://hydril.la/', 'mapping', root_mapping_id) + 'https://hydril.la/', 'mapping', variant['root_mapping_id']) fetched = lambda d: 'Fetching ' not in containers['dialog_container'].text WebDriverWait(driver, 10).until(fetched) @@ -203,7 +234,6 @@ def test_install_normal_usage(driver, execute_in_page, complex_variant): 'indexeddb_error_file_uses', 'failure_to_communicate_fetch', 'HTTP_code_file', - 'not_valid_text', 'sha256_mismatch', 'indexeddb_error_write' ]) @@ -243,7 +273,7 @@ def test_install_dialogs(driver, execute_in_page, message): if message == 'fetching_data': execute_in_page( ''' - browser.tabs.sendMessage = () => new Promise(cb => {}); + window.mock_cacher_fetch = () => new Promise(cb => {}); install_view.show(...arguments); ''', 'https://hydril.la/', 'mapping', 'mapping-a') @@ -253,7 +283,8 @@ def test_install_dialogs(driver, execute_in_page, message): elif message == 'failure_to_communicate_sendmessage': execute_in_page( ''' - browser.tabs.sendMessage = () => Promise.resolve({error: "sth"}); + window.mock_cacher_fetch = + () => {throw new Error("Something happened :o")}; install_view.show(...arguments); ''', 'https://hydril.la/', 'mapping', 'mapping-a') @@ -262,8 +293,8 @@ def test_install_dialogs(driver, execute_in_page, message): elif message == 'HTTP_code_item': execute_in_page( ''' - const response = {ok: false, status: 404}; - browser.tabs.sendMessage = () => Promise.resolve(response); + const response = new Response("", {status: 404}); + window.mock_cacher_fetch = () => Promise.resolve(response); install_view.show(...arguments); ''', 'https://hydril.la/', 'mapping', 'mapping-a') @@ -272,8 +303,8 @@ def test_install_dialogs(driver, execute_in_page, message): elif message == 'invalid_JSON': execute_in_page( ''' - const response = {ok: true, status: 200, error_json: "sth"}; - browser.tabs.sendMessage = () => Promise.resolve(response); + const response = new Response("sth", {status: 200}); + window.mock_cacher_fetch = () => Promise.resolve(response); install_view.show(...arguments); ''', 'https://hydril.la/', 'mapping', 'mapping-a') @@ -282,12 +313,11 @@ def test_install_dialogs(driver, execute_in_page, message): elif message == 'newer_API_version': execute_in_page( ''' - const old_sendMessage = browser.tabs.sendMessage; - browser.tabs.sendMessage = async function(...args) { - const response = await old_sendMessage(...args); - response.json.$schema = "https://hydrilla.koszko.org/schemas/api_mapping_description-255.1.schema.json"; - return response; - } + const newer_schema_url = + "https://hydrilla.koszko.org/schemas/api_mapping_description-255.1.schema.json"; + const mocked_json_data = JSON.stringify({$schema: newer_schema_url}); + const response = new Response(mocked_json_data, {status: 200}); + window.mock_cacher_fetch = () => Promise.resolve(response); install_view.show(...arguments); ''', 'https://hydril.la/', 'mapping', 'mapping-a', [2022, 5, 10]) @@ -297,12 +327,18 @@ def test_install_dialogs(driver, execute_in_page, message): elif message == 'invalid_response_format': execute_in_page( ''' - const old_sendMessage = browser.tabs.sendMessage; - browser.tabs.sendMessage = async function(...args) { - const response = await old_sendMessage(...args); - /* identifier is not a string as it should be. */ - response.json.identifier = 1234567; - return response; + window.mock_cacher_fetch = async function(...args) { + const response = await fetch(...args); + const json = await response.json(); + + /* identifier is no longer a string as it should be. */ + json.identifier = 1234567; + + return new Response(JSON.stringify(json), { + status: response.status, + statusText: response.statusText, + headers: [...response.headers.entries()] + }); } install_view.show(...arguments); ''', @@ -352,7 +388,7 @@ def test_install_dialogs(driver, execute_in_page, message): elif message == 'failure_to_communicate_fetch': execute_in_page( ''' - fetch = () => {throw "some error";}; + fetch = () => {throw new Error("some error");}; returnval(install_view.show(...arguments)); ''', 'https://hydril.la/', 'mapping', 'mapping-b') @@ -372,18 +408,6 @@ def test_install_dialogs(driver, execute_in_page, message): execute_in_page('returnval(install_view.install_but);').click() assert_dlg(['conf_buts'], 'Repository sent HTTP code 400 :(') - elif message == 'not_valid_text': - execute_in_page( - ''' - const err = () => {throw "some error";}; - fetch = () => Promise.resolve({ok: true, status: 200, text: err}); - returnval(install_view.show(...arguments)); - ''', - 'https://hydril.la/', 'mapping', 'mapping-b') - - execute_in_page('returnval(install_view.install_but);').click() - - assert_dlg(['conf_buts'], "Repository's response is not valid text :(") elif message == 'sha256_mismatch': execute_in_page( ''' diff --git a/test/haketilo_test/unit/test_policy_deciding.py b/test/haketilo_test/unit/test_policy_deciding.py index 75b35ac..1be488f 100644 --- a/test/haketilo_test/unit/test_policy_deciding.py +++ b/test/haketilo_test/unit/test_policy_deciding.py @@ -23,19 +23,36 @@ import pytest from ..script_loader import load_script -csp_re = re.compile(r'^\S+\s+\S+;(?:\s+\S+\s+\S+;)*$') -rule_re = re.compile(r'^\s*(?P<src_kind>\S+)\s+(?P<allowed_origins>\S+)$') +csp_re = re.compile(r''' +^ +\S+(?:\s+\S+)+; # first directive +(?: + \s+\S+(?:\s+\S+)+; # subsequent directive +)* +$ +''', +re.VERBOSE) + +rule_re = re.compile(r''' +^ +\s* +(?P<src_kind>\S+) +\s+ +(?P<allowed_origins> + \S+(?:\s+\S+)* +) +$ +''', re.VERBOSE) + def parse_csp(csp): - ''' - Parsing of CSP string into a dict. A simplified format of CSP is assumed. - ''' + '''Parsing of CSP string into a dict.''' assert csp_re.match(csp) result = {} for rule in csp.split(';')[:-1]: match = rule_re.match(rule) - result[match.group('src_kind')] = match.group('allowed_origins') + result[match.group('src_kind')] = match.group('allowed_origins').split() return result @@ -78,10 +95,10 @@ def test_decide_policy(execute_in_page): for prop in ('mapping', 'payload', 'nonce', 'error'): assert prop not in policy assert parse_csp(policy['csp']) == { - 'prefetch-src': "'none'", - 'script-src-attr': "'none'", - 'script-src': "'none'", - 'script-src-elem': "'none'" + 'prefetch-src': ["'none'"], + 'script-src-attr': ["'none'"], + 'script-src': ["'none'", "'unsafe-eval'"], + 'script-src-elem': ["'none'"] } policy = execute_in_page( @@ -95,10 +112,10 @@ def test_decide_policy(execute_in_page): for prop in ('payload', 'nonce', 'error'): assert prop not in policy assert parse_csp(policy['csp']) == { - 'prefetch-src': "'none'", - 'script-src-attr': "'none'", - 'script-src': "'none'", - 'script-src-elem': "'none'" + 'prefetch-src': ["'none'"], + 'script-src-attr': ["'none'"], + 'script-src': ["'none'", "'unsafe-eval'"], + 'script-src-elem': ["'none'"] } policy = execute_in_page( @@ -114,10 +131,10 @@ def test_decide_policy(execute_in_page): assert policy['nonce'] == \ sha256('m1:res1:http://kno.wn/:abcd'.encode()).digest().hex() assert parse_csp(policy['csp']) == { - 'prefetch-src': f"'none'", - 'script-src-attr': f"'none'", - 'script-src': f"'nonce-{policy['nonce']}'", - 'script-src-elem': f"'nonce-{policy['nonce']}'" + 'prefetch-src': ["'none'"], + 'script-src-attr': ["'none'"], + 'script-src': [f"'nonce-{policy['nonce']}'", "'unsafe-eval'"], + 'script-src-elem': [f"'nonce-{policy['nonce']}'"] } policy = execute_in_page( @@ -128,8 +145,8 @@ def test_decide_policy(execute_in_page): for prop in ('mapping', 'payload', 'nonce'): assert prop not in policy assert parse_csp(policy['csp']) == { - 'prefetch-src': "'none'", - 'script-src-attr': "'none'", - 'script-src': "'none'", - 'script-src-elem': "'none'" + 'prefetch-src': ["'none'"], + 'script-src-attr': ["'none'"], + 'script-src': ["'none'", "'unsafe-eval'"], + 'script-src-elem': ["'none'"] } diff --git a/test/haketilo_test/unit/test_policy_enforcing.py b/test/haketilo_test/unit/test_policy_enforcing.py index bbc3eb9..4bc6470 100644 --- a/test/haketilo_test/unit/test_policy_enforcing.py +++ b/test/haketilo_test/unit/test_policy_enforcing.py @@ -31,15 +31,19 @@ nonce = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' allow_policy = {'allow': True} block_policy = { 'allow': False, - 'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'none'; script-src-elem 'none'; frame-src http://* https://*;" + 'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'none' 'unsafe-eval'; script-src-elem 'none'; frame-src http://* https://*;" } payload_policy = { 'mapping': 'somemapping', 'payload': {'identifier': 'someresource'}, - 'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'nonce-{nonce}'; script-src-elem 'nonce-{nonce}';" + 'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'nonce-{nonce}' 'unsafe-eval'; script-src-elem 'nonce-{nonce}';" } -content_script = load_script('content/policy_enforcing.js') + ''';{ +def content_script(): + return load_script('content/policy_enforcing.js') + \ + content_script_appended_code + +content_script_appended_code = ''';{ const smuggled_what_to_do = /^[^#]*#?(.*)$/.exec(document.URL)[1]; const what_to_do = smuggled_what_to_do === "" ? {policy: {allow: true}} : JSON.parse(decodeURIComponent(smuggled_what_to_do)); diff --git a/test/haketilo_test/unit/test_popup.py b/test/haketilo_test/unit/test_popup.py index e62feb7..3ef7906 100644 --- a/test/haketilo_test/unit/test_popup.py +++ b/test/haketilo_test/unit/test_popup.py @@ -81,28 +81,18 @@ mocked_page_infos = { tab_mock_js = ''' ; const mocked_page_info = (%s)[/#mock_page_info-(.*)$/.exec(document.URL)[1]]; +const old_sendMessage = browser.tabs.sendMessage; browser.tabs.sendMessage = async function(tab_id, msg) { const this_tab_id = (await browser.tabs.getCurrent()).id; if (tab_id !== this_tab_id) throw `not current tab id (${tab_id} instead of ${this_tab_id})`; - if (msg[0] === "page_info") { + if (msg[0] === "page_info") return mocked_page_info; - } else if (msg[0] === "repo_query") { - const response = await fetch(msg[1]); - if (!response) - return {error: "Something happened :o"}; - - const result = {ok: response.ok, status: response.status}; - try { - result.json = await response.json(); - } catch(e) { - result.error_json = "" + e; - } - return result; - } else { + else if (msg[0] === "repo_query") + return old_sendMessage(tab_id, msg); + else throw `bad sendMessage message type: '${msg[0]}'`; - } } const old_tabs_query = browser.tabs.query; @@ -113,6 +103,8 @@ browser.tabs.query = async function(query) { } ''' % json.dumps(mocked_page_infos) +tab_mock_js = mock_cacher_code + tab_mock_js + popup_ext_data = { 'background_script': broker_js, 'extra_html': ExtraHTML( diff --git a/test/haketilo_test/unit/test_repo_query.py b/test/haketilo_test/unit/test_repo_query.py index f6cae93..c785406 100644 --- a/test/haketilo_test/unit/test_repo_query.py +++ b/test/haketilo_test/unit/test_repo_query.py @@ -29,7 +29,7 @@ repo_urls = [f'https://hydril.la/{s}' for s in ('', '1/', '2/', '3/', '4/')] queried_url = 'https://example_a.com/something' def setup_view(execute_in_page, repo_urls, tab={'id': 0}): - mock_cacher(execute_in_page) + execute_in_page(mock_cacher_code) execute_in_page(load_script('html/repo_query.js')) execute_in_page( @@ -185,8 +185,10 @@ def test_repo_query_messages(driver, execute_in_page, message): elif message == 'failure_to_communicate': setup_view(execute_in_page, repo_urls) execute_in_page( - 'browser.tabs.sendMessage = () => Promise.resolve({error: "sth"});' - ) + ''' + window.mock_cacher_fetch = + () => {throw new Error("Something happened :o")}; + ''') show_and_wait_for_repo_entry() elem = execute_in_page('returnval(view.repo_entries[0].info_div);') @@ -196,8 +198,8 @@ def test_repo_query_messages(driver, execute_in_page, message): setup_view(execute_in_page, repo_urls) execute_in_page( ''' - const response = {ok: false, status: 405}; - browser.tabs.sendMessage = () => Promise.resolve(response); + const response = new Response("", {status: 405}); + window.mock_cacher_fetch = () => Promise.resolve(response); ''') show_and_wait_for_repo_entry() @@ -208,8 +210,8 @@ def test_repo_query_messages(driver, execute_in_page, message): setup_view(execute_in_page, repo_urls) execute_in_page( ''' - const response = {ok: true, status: 200, error_json: "sth"}; - browser.tabs.sendMessage = () => Promise.resolve(response); + const response = new Response("sth", {status: 200}); + window.mock_cacher_fetch = () => Promise.resolve(response); ''') show_and_wait_for_repo_entry() @@ -220,12 +222,11 @@ def test_repo_query_messages(driver, execute_in_page, message): setup_view(execute_in_page, repo_urls) execute_in_page( ''' - const response = { - ok: true, - status: 200, - json: {$schema: "https://hydrilla.koszko.org/schemas/api_query_result-255.2.1.schema.json"} - }; - browser.tabs.sendMessage = () => Promise.resolve(response); + const newer_schema_url = + "https://hydrilla.koszko.org/schemas/api_query_result-255.2.1.schema.json"; + const mocked_json_data = JSON.stringify({$schema: newer_schema_url}); + const response = new Response(mocked_json_data, {status: 200}); + window.mock_cacher_fetch = () => Promise.resolve(response); ''') show_and_wait_for_repo_entry() @@ -236,13 +237,19 @@ def test_repo_query_messages(driver, execute_in_page, message): setup_view(execute_in_page, repo_urls) execute_in_page( ''' - const response = { - ok: true, - status: 200, - /* $schema is not a string as it should be. */ - json: {$schema: null} - }; - browser.tabs.sendMessage = () => Promise.resolve(response); + window.mock_cacher_fetch = async function(...args) { + const response = await fetch(...args); + const json = await response.json(); + + /* $schema is no longer a string as it should be. */ + json.$schema = null; + + return new Response(JSON.stringify(json), { + status: response.status, + statusText: response.statusText, + headers: [...response.headers.entries()] + }); + } ''') show_and_wait_for_repo_entry() @@ -252,7 +259,7 @@ def test_repo_query_messages(driver, execute_in_page, message): elif message == 'querying_repo': setup_view(execute_in_page, repo_urls) execute_in_page( - 'browser.tabs.sendMessage = () => new Promise(() => {});' + 'window.mock_cacher_fetch = () => new Promise(cb => {});' ) show_and_wait_for_repo_entry() @@ -262,15 +269,12 @@ def test_repo_query_messages(driver, execute_in_page, message): setup_view(execute_in_page, repo_urls) execute_in_page( ''' - const response = { - ok: true, - status: 200, - json: { - $schema: "https://hydrilla.koszko.org/schemas/api_query_result-1.schema.json", - mappings: [] - } - }; - browser.tabs.sendMessage = () => Promise.resolve(response); + const schema_url = + "https://hydrilla.koszko.org/schemas/api_query_result-1.schema.json"; + const mocked_json_data = + JSON.stringify({$schema: schema_url, mappings: []}); + const response = new Response(mocked_json_data, {status: 200}); + window.mock_cacher_fetch = () => Promise.resolve(response); ''') show_and_wait_for_repo_entry() diff --git a/test/haketilo_test/unit/test_repo_query_cacher.py b/test/haketilo_test/unit/test_repo_query_cacher.py index 5fbc5cd..3f0a00d 100644 --- a/test/haketilo_test/unit/test_repo_query_cacher.py +++ b/test/haketilo_test/unit/test_repo_query_cacher.py @@ -85,34 +85,35 @@ def run_content_script_in_new_window(driver, url): 'background_script': lambda: bypass_js() + ';' + tab_id_responder }) @pytest.mark.usefixtures('webextension') -def test_repo_query_cacher_normal_use(driver, execute_in_page): +def test_repo_query_cacher_normal_use(driver): """ Test if HTTP requests made through our cacher return correct results. """ tab_id = run_content_script_in_new_window(driver, 'https://gotmyowndoma.in') result = fetch_through_cache(driver, tab_id, 'https://counterdoma.in/') - assert set(result.keys()) == {'ok', 'status', 'json'} - counter_initial = result['json']['counter'] + assert set(result.keys()) == {'status', 'statusText', 'headers', 'body'} + counter_initial = json.loads(bytes.fromhex(result['body']))['counter'] assert type(counter_initial) is int for i in range(2): result = fetch_through_cache(driver, tab_id, 'https://counterdoma.in/') - assert result['json']['counter'] == counter_initial + assert json.loads(bytes.fromhex(result['body'])) \ + == {'counter': counter_initial} tab_id = run_content_script_in_new_window(driver, 'https://gotmyowndoma.in') result = fetch_through_cache(driver, tab_id, 'https://counterdoma.in/') - assert result['json']['counter'] == counter_initial + 1 + assert json.loads(bytes.fromhex(result['body'])) \ + == {'counter': counter_initial + 1} for i in range(2): result = fetch_through_cache(driver, tab_id, 'https://nxdoma.in/') - assert set(result.keys()) == {'ok', 'status', 'error_json'} - assert result['ok'] == False assert result['status'] == 404 for i in range(2): result = fetch_through_cache(driver, tab_id, 'bad://url') assert set(result.keys()) == {'error'} + assert result['error']['name'] == 'TypeError' @pytest.mark.ext_data({ 'content_script': content_script, @@ -128,3 +129,7 @@ def test_repo_query_cacher_bgscript_error(driver): result = fetch_through_cache(driver, tab_id, 'https://counterdoma.in/') assert set(result.keys()) == {'error'} + assert set(result['error'].keys()) == \ + {'name', 'message', 'fileName', 'lineNumber'} + assert result['error']['message'] == \ + "Couldn't communicate with background script." diff --git a/test/haketilo_test/unit/test_webrequest.py b/test/haketilo_test/unit/test_webrequest.py index 1244117..dc329b8 100644 --- a/test/haketilo_test/unit/test_webrequest.py +++ b/test/haketilo_test/unit/test_webrequest.py @@ -85,7 +85,7 @@ nonce = f'nonce-{sha256(nonce_source).digest().hex()}' payload_csp_header = { 'name': f'Content-Security-Policy', 'value': ("prefetch-src 'none'; script-src-attr 'none'; " - f"script-src '{nonce}'; script-src-elem '{nonce}';") + f"script-src '{nonce}' 'unsafe-eval'; script-src-elem '{nonce}';") } sample_payload_headers = [ @@ -107,7 +107,7 @@ sample_blocked_headers.append(sample_csp_header) sample_blocked_headers.append({ 'name': f'Content-Security-Policy', 'value': ("prefetch-src 'none'; script-src-attr 'none'; " - f"script-src 'none'; script-src-elem 'none';") + "script-src 'none' 'unsafe-eval'; script-src-elem 'none';") }) @pytest.mark.get_page('https://gotmyowndoma.in') diff --git a/test/haketilo_test/unit/utils.py b/test/haketilo_test/unit/utils.py index 7ddf92a..9b3e4a0 100644 --- a/test/haketilo_test/unit/utils.py +++ b/test/haketilo_test/unit/utils.py @@ -228,12 +228,21 @@ def are_scripts_allowed(driver, nonce=None): return driver.execute_script( ''' document.haketilo_scripts_allowed = false; + document.haketilo_eval_allowed = false; const html_ns = "http://www.w3.org/1999/xhtml"; const script = document.createElementNS(html_ns, "script"); - script.innerHTML = "document.haketilo_scripts_allowed = true;"; + script.innerHTML = ` + document.haketilo_scripts_allowed = true; + eval('document.haketilo_eval_allowed = true;'); + `; if (arguments[0]) script.setAttribute("nonce", arguments[0]); (document.head || document.documentElement).append(script); + + if (document.haketilo_scripts_allowed != + document.haketilo_eval_allowed) + throw "scripts allowed but eval blocked"; + return document.haketilo_scripts_allowed; ''', nonce) @@ -246,36 +255,40 @@ def mock_broadcast(execute_in_page): 'Object.keys(broadcast).forEach(k => broadcast[k] = () => {});' ) -def mock_cacher(execute_in_page): - """ - Some parts of code depend on content/repo_query_cacher.js and - background/CORS_bypass_server.js running in their appropriate contexts. This - function modifies the relevant browser.runtime.sendMessage function to - perform fetch(), bypassing the cacher. - """ - execute_in_page( - '''{ - const old_sendMessage = browser.tabs.sendMessage, old_fetch = fetch; - async function new_sendMessage(tab_id, msg) { - if (msg[0] !== "repo_query") - return old_sendMessage(tab_id, msg); - - /* Use snapshotted fetch(), allow other test code to override it. */ - const response = await old_fetch(msg[1]); - if (!response) - return {error: "Something happened :o"}; - - const result = {ok: response.ok, status: response.status}; - try { - result.json = await response.json(); - } catch(e) { - result.error_json = "" + e; - } - return result; - } +""" +Some parts of code depend on content/repo_query_cacher.js and +background/CORS_bypass_server.js running in their appropriate contexts. This +snippet modifies the relevant browser.runtime.sendMessage function to perform +fetch(), bypassing the cacher. +""" +mock_cacher_code = '''{ +const uint8_to_hex = + array => [...array].map(b => ("0" + b.toString(16)).slice(-2)).join(""); - browser.tabs.sendMessage = new_sendMessage; - }''') +const old_sendMessage = browser.tabs.sendMessage; +window.mock_cacher_fetch = fetch; +browser.tabs.sendMessage = async function(tab_id, msg) { + if (msg[0] !== "repo_query") + return old_sendMessage(tab_id, msg); + + /* + * Use snapshotted fetch() under the name window.mock_cacher_fetch, + * allow other test code to override it. + */ + try { + const response = await window.mock_cacher_fetch(msg[1]); + const buf = await response.arrayBuffer(); + return { + status: response.status, + statusText: response.statusText, + headers: [...response.headers.entries()], + body: uint8_to_hex(new Uint8Array(buf)) + } + } catch(e) { + return {error: {name: e.name, message: e.message}}; + } +} +}''' """ Convenience snippet of code to retrieve a copy of given object with only those diff --git a/test/haketilo_test/world_wide_library.py b/test/haketilo_test/world_wide_library.py index 1a90c42..92ce97e 100644 --- a/test/haketilo_test/world_wide_library.py +++ b/test/haketilo_test/world_wide_library.py @@ -33,6 +33,8 @@ from shutil import rmtree from threading import Lock from uuid import uuid4 import json +import functools as ft +import operator as op from .misc_constants import here from .unit.utils import * # sample repo data @@ -114,7 +116,7 @@ sample_contents = [f'Mi povas manĝi vitron, ĝi ne damaĝas min {i}' for i in range(9)] sample_hashes = [sha256(c.encode()).digest().hex() for c in sample_contents] -file_url = lambda hashed: f'https://hydril.la/file/sha256/{hashed}' +file_url = ft.partial(op.concat, 'https://hydril.la/file/sha256/') sample_files_catalog = dict([(file_url(h), make_handler(c)) for h, c in zip(sample_hashes, sample_contents)]) @@ -144,18 +146,37 @@ for i in range(10): 'dependencies': [] }) +# The one below will generate items with schema still at version 1, so required +# mappings will be ignored. +sample_resource_templates.append({ + 'id_suffix': 'a-w-required-mapping-v1', + 'files_count': 1, + 'dependencies': [], + 'required_mappings': [{'identifier': 'mapping-a'}], + 'include_in_query': False +}) + +sample_resource_templates.append({ + 'id_suffix': 'a-w-required-mapping-v2', + 'files_count': 1, + 'dependencies': [], + 'required_mappings': [{'identifier': 'mapping-a'}], + 'schema_ver': '2', + 'include_in_query': False +}) + sample_resources_catalog = {} sample_mappings_catalog = {} sample_queries = {} for srt in sample_resource_templates: resource = make_sample_resource() - resource['identifier'] = f'resource-{srt["id_suffix"]}' - resource['long_name'] = resource['identifier'].upper() - resource['uuid'] = str(uuid4()) - resource['dependencies'] = srt['dependencies'] - resource['source_copyright'] = [] - resource['scripts'] = [] + resource['identifier'] = f'resource-{srt["id_suffix"]}' + resource['long_name'] = resource['identifier'].upper() + resource['uuid'] = str(uuid4()) + resource['dependencies'] = srt['dependencies'] + resource['source_copyright'] = [] + resource['scripts'] = [] for i in range(srt['files_count']): file_ref = {'file': f'file_{i}', 'sha256': sample_hashes[i]} resource[('source_copyright', 'scripts')[i & 1]].append(file_ref) @@ -174,22 +195,25 @@ for srt in sample_resource_templates: sufs = [srt["id_suffix"], *[l for l in srt["id_suffix"] if l.isalpha()]] patterns = [f'https://example_{suf}.com/*' for suf in set(sufs)] - payloads = {} + mapping['payloads'] = {} for pat in patterns: - payloads[pat] = {'identifier': resource['identifier']} + mapping['payloads'][pat] = {'identifier': resource['identifier']} - queryable_url = pat.replace('*', 'something') - if queryable_url not in sample_queries: - sample_queries[queryable_url] = [] + if not srt.get('include_in_query', True): + continue - sample_queries[queryable_url].append({ + sample_queries.setdefault(pat.replace('*', 'something'), []).append({ 'identifier': mapping['identifier'], 'long_name': mapping['long_name'], 'version': mapping_versions[1] }) - mapping['payloads'] = payloads + for item in resource, mapping: + if 'required_mappings' in srt: + item['required_mappings'] = srt['required_mappings'] + if 'schema_ver' in srt: + item['$schema'] = item['$schema'].replace('1', srt['schema_ver']) for item, versions, catalog in [ (resource, resource_versions, sample_resources_catalog), |