add new root content script

10 files changed, 275 insertions, 40 deletions

diff --git a/background/patterns_query_manager.js b/background/patterns_query_manager.js
index 78cd0ef..3b74ee9 100644
--- a/background/patterns_query_manager.js
+++ b/background/patterns_query_manager.js
@@ -49,6 +49,9 @@
 #FROM common/browser.js IMPORT browser
 #ENDIF
 
+let default_allow = {};
+#EXPORT default_allow
+
 let secret;
 
 const tree = pqt.make();
@@ -72,8 +75,9 @@ async function update_content_script()
 	script_update_needed = false;
 
 	const code = `\
-this.haketilo_secret       = ${secret};
-this.haketilo_pattern_tree = ${JSON.stringify(tree)};
+this.haketilo_secret        = ${JSON.stringify(secret)};
+this.haketilo_pattern_tree  = ${JSON.stringify(tree)};
+this.haketilo_default_allow = ${JSON.stringify(default_allow.value)};
 if (this.haketilo_content_script_main)
     haketilo_content_script_main();`;
 
@@ -151,6 +155,14 @@ async function start(secret_)
     initial_mappings.forEach(m => register("mappings", m));
     initial_blocking.forEach(b => register("blocking", b));
 
+    const set_allow_val = ch => default_allow.value = (ch.new_val || {}).value;
+    const [setting_tracking, initial_settings] =
+	  await haketilodb.track.settings(set_allow_val);
+    for (const setting of initial_settings) {
+	if (setting.name === "default_allow")
+	    Object.assign(default_allow, setting);
+    }
+
 #IF MOZILLA || MV3
     script_update_needed = true;
     await update_content_script();
diff --git a/background/webrequest.js b/background/webrequest.js
index 891cac3..bd091dc 100644
--- a/background/webrequest.js
+++ b/background/webrequest.js
@@ -50,22 +50,10 @@
 #FROM common/misc.js    IMPORT is_privileged_url, csp_header_regex
 #FROM common/policy.js  IMPORT decide_policy
 
-#FROM background/patterns_query_manager.js IMPORT tree
+#FROM background/patterns_query_manager.js IMPORT tree, default_allow
 
 let secret;
 
-let default_allow = false;
-
-async function track_default_allow()
-{
-    const set_val = ch => default_allow = (ch.new_val || {}).value;
-    const [tracking, settings] = await haketilodb.track.settings(set_val);
-    for (const setting of settings) {
-	if (setting.name === "default_allow")
-	    default_allow = setting.value;
-    }
-}
-
 function on_headers_received(details)
 {
     const url = details.url;
@@ -74,7 +62,8 @@ function on_headers_received(details)
 
     let headers = details.responseHeaders;
 
-    const policy = decide_policy(tree, details.url, !!default_allow, secret);
+    const policy =
+	  decide_policy(tree, details.url, !!default_allow.value, secret);
     if (policy.allow)
 	return;
 
diff --git a/common/misc.js b/common/misc.js
index ed8f400..f8e0812 100644
--- a/common/misc.js
+++ b/common/misc.js
@@ -96,9 +96,12 @@ function open_in_settings(prefix, name)
  * Check if url corresponds to a browser's special page (or a directory index in
  * case of `file://' protocol).
  */
-const privileged_reg =
-      /^(chrome(-extension)?|moz-extension):\/\/|^about:|^file:\/\/.*\/$/;
-#EXPORT  url => privileged_reg.test(url)  AS is_privileged_url
+#IF MOZILLA
+const priv_reg = /^moz-extension:\/\/|^about:|^file:\/\/[^?#]*\/([?#]|$)/;
+#ELIF CHROMIUM
+const priv_reg = /^chrome(-extension)?:\/\/|^about:|^file:\/\/[^?#]*\/([?#]|$)/;
+#ENDIF
+#EXPORT  url => priv_reg.test(url)  AS is_privileged_url
 
 /* Parse a CSP header */
 function parse_csp(csp) {
diff --git a/content/content.js b/content/content.js
new file mode 100644
index 0000000..804a473
--- /dev/null
+++ b/content/content.js
@@ -0,0 +1,89 @@
+/**
+ * This file is part of Haketilo.
+ *
+ * Function: Content scripts - main script.
+ *
+ * Copyright (C) 2022 Wojtek Kosior
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * As additional permission under GNU GPL version 3 section 7, you
+ * may distribute forms of that code without the copy of the GNU
+ * GPL normally required by section 4, provided you include this
+ * license notice and, in case of non-source distribution, a URL
+ * through which recipients can access the Corresponding Source.
+ * If you modify file(s) with this exception, you may extend this
+ * exception to your version of the file(s), but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * As a special exception to the GPL, any HTML file which merely
+ * makes function calls to this code, and for that purpose
+ * includes it by reference shall be deemed a separate work for
+ * copyright law purposes. If you modify this code, you may extend
+ * this exception to your version of the code, but you are not
+ * obligated to do so. If you do not wish to do so, delete this
+ * exception statement from your version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * I, Wojtek Kosior, thereby promise not to sue for violation of this file's
+ * license. Although I request that you do not make use of this code in a
+ * proprietary program, I am not going to enforce this in court.
+ */
+
+#IMPORT content/repo_query_cacher.js
+
+#FROM common/browser.js           IMPORT browser
+#FROM common/misc.js              IMPORT is_privileged_url
+#FROM common/policy.js            IMPORT decide_policy
+#FROM content/policy_enforcing.js IMPORT enforce_blocking
+
+let already_run = false, page_info;
+
+function on_page_info_request([type], sender, respond_cb) {
+    if (type !== "page_info")
+	return;
+
+    respond_cb(page_info);
+}
+
+globalThis.haketilo_content_script_main = function() {
+    if (already_run)
+        return;
+
+    already_run = true;
+
+    if (is_privileged_url(document.URL))
+	return;
+
+    browser.runtime.onMessage.addListener(on_page_info_request);
+    repo_query_cacher.start();
+
+    const policy = decide_policy(globalThis.haketilo_pattern_tree,
+				 document.URL,
+				 globalThis.haketilo_defualt_allow,
+				 globalThis.haketilo_secret);
+    page_info = Object.assign({url: document.URL}, policy);
+    ["csp", "nonce"].forEach(prop => delete page_info[prop]);
+
+    enforce_blocking(policy);
+}
+
+function main() {
+    if (globalThis.haketilo_pattern_tree !== undefined)
+	globalThis.haketilo_content_script_main();
+}
+
+#IF !UNIT_TEST
+main();
+#ENDIF
diff --git a/html/default_blocking_policy.js b/html/default_blocking_policy.js
index 758572f..b0577f7 100644
--- a/html/default_blocking_policy.js
+++ b/html/default_blocking_policy.js
@@ -83,6 +83,6 @@ async function init_default_policy_dialog()
     toggle_policy_but.addEventListener("click", toggle_policy);
 }
 
-#IF !TEST_UNIT
+#IF !UNIT_TEST
 init_default_policy_dialog();
 #ENDIF
diff --git a/html/popup.js b/html/popup.js
index 1efc4b0..b4602f3 100644
--- a/html/popup.js
+++ b/html/popup.js
@@ -88,8 +88,8 @@ function show_page_info(page_info) {
 	    var mapping = `None (scripts ${scripts_fate} by a rule)`;
 	else if (page_info.mapping)
 	    var mapping = page_info.mapping;
-	else
-	    var mapping = `None (scripts ${scripts_fate} by default policy)`;
+	else if (page_info.error)
+	    var mapping = `None (error occured when determining policy)`;
 	by_id("mapping_used").innerText = mapping;
     }
 }
diff --git a/test/unit/test_content.py b/test/unit/test_content.py
new file mode 100644
index 0000000..c8e0987
--- /dev/null
+++ b/test/unit/test_content.py
@@ -0,0 +1,119 @@
+# SPDX-License-Identifier: CC0-1.0
+
+"""
+Haketilo unit tests - main content script
+"""
+
+# This file is part of Haketilo
+#
+# Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the CC0 1.0 Universal License as published by
+# the Creative Commons Corporation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# CC0 1.0 Universal License for more details.
+
+import pytest
+import json
+from selenium.webdriver.support.ui import WebDriverWait
+
+from ..script_loader import load_script
+
+# From:
+# https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/contentScripts/register
+# it is unclear whether the dynamically-registered content script is guaranteed
+# to be always executed after statically-registered ones. We want to test both
+# cases, so we'll make the mocked dynamic content script execute before
+# content.js on http:// pages and after it on https:// pages.
+dynamic_script = \
+    ''';
+    this.haketilo_secret        = "abracadabra";
+    this.haketilo_pattern_tree  = {};
+    this.haketilo_defualt_allow = false;
+
+    if (this.haketilo_content_script_main)
+        this.haketilo_content_script_main();
+    '''
+
+content_script = \
+    '''
+    /* Mock dynamic content script - case 'before'. */
+    if (/#dynamic_before$/.test(document.URL)) {
+        %s;
+    }
+
+    /* Place amalgamated content.js here. */
+    %s;
+
+    /* Rest of mocks */
+    const data_to_verify = {};
+    function data_set(prop, val) {
+        data_to_verify[prop] = val;
+        window.wrappedJSObject.data_to_verify = JSON.stringify(data_to_verify);
+    }
+
+    repo_query_cacher.start = () => data_set("cacher_started", true);
+
+    enforce_blocking = policy => data_set("enforcing", policy);
+
+    browser.runtime.onMessage.addListener = async function (listener_cb) {
+        await new Promise(cb => setTimeout(cb, 0));
+
+        /* Mock a good request. */
+        const set_good = val => data_set("good_request_result", val);
+        listener_cb(["page_info"], {}, val => set_good(val));
+
+        /* Mock a bad request. */
+        const set_bad = val => data_set("bad_request_result", val);
+        listener_cb(["???"], {}, val => set_bad(val));
+    }
+
+    /* main() call - normally present in content.js, inside '#IF !UNIT_TEST'. */
+    main();
+
+    /* Mock dynamic content script - case 'after'. */
+    if (/#dynamic_after$/.test(document.URL)) {
+        %s;
+    }
+
+    data_set("script_run_without_errors", true);
+    ''' % (dynamic_script, load_script('content/content.js'), dynamic_script)
+
+@pytest.mark.ext_data({'content_script': content_script})
+@pytest.mark.usefixtures('webextension')
+@pytest.mark.parametrize('target', ['dynamic_before', 'dynamic_after'])
+def test_content_unprivileged_page(driver, execute_in_page, target):
+    """
+    Test functioning of content.js on an page using unprivileged schema (e.g.
+    'https://' and not 'about:').
+    """
+    driver.get(f'https://gotmyowndoma.in/index.html#{target}')
+    data = json.loads(driver.execute_script('return window.data_to_verify;'))
+
+    assert 'gotmyowndoma.in' in data['good_request_result']['url']
+    assert 'bad_request_result' not in data
+
+    assert data['cacher_started'] == True
+
+    assert data['enforcing']['allow']  == False
+    assert 'mapping' not in data['enforcing']
+    assert 'error'   not in data['enforcing']
+
+    assert data['script_run_without_errors'] == True
+
+@pytest.mark.ext_data({'content_script': content_script})
+@pytest.mark.usefixtures('webextension')
+@pytest.mark.parametrize('target', ['dynamic_before', 'dynamic_after'])
+def test_content_privileged_page(driver, execute_in_page, target):
+    """
+    Test functioning of content.js on an page considered privileged (e.g. a
+    directory listing at 'file:///').
+    """
+    driver.get(f'file:///#{target}')
+    data = json.loads(driver.execute_script('return window.data_to_verify;'))
+
+    assert data == {'script_run_without_errors': True}
diff --git a/test/unit/test_patterns_query_manager.py b/test/unit/test_patterns_query_manager.py
index 4e6d1bf..c6ebb81 100644
--- a/test/unit/test_patterns_query_manager.py
+++ b/test/unit/test_patterns_query_manager.py
@@ -18,7 +18,6 @@ Haketilo unit tests - building pattern tree and putting it in a content script
 # CC0 1.0 Universal License for more details.
 
 import pytest
-import re
 import json
 from selenium.webdriver.support.ui import WebDriverWait
 
@@ -35,13 +34,19 @@ def simple_sample_mapping(patterns, fruit):
         'payloads': payloads
     }
 
-content_script_tree_re = re.compile(r'this.haketilo_pattern_tree = (.*);')
-def extract_tree_data(content_script_text):
-    return json.loads(content_script_tree_re.search(content_script_text)[1])
-
-content_script_mapping_re = re.compile(r'this.haketilo_mappings = (.*);')
-def extract_mappings_data(content_script_text):
-    return json.loads(content_script_mapping_re.search(content_script_text)[1])
+def get_content_script_values(driver, content_script):
+    """
+    Allow easy extraction of 'this.something = ...' values from generated
+    content script and verify the content script is syntactically correct.
+    """
+    return driver.execute_script(
+        '''
+        function value_holder() {
+            %s;
+            return this;
+        }
+        return value_holder.call({});
+        ''' % content_script)
 
 # Fields that are not relevant for testing are omitted from these mapping
 # definitions.
@@ -87,7 +92,7 @@ def test_pqm_tree_building(driver, execute_in_page):
     execute_in_page(
         '''
         const [initial_mappings, initial_blocking] = arguments.slice(0, 2);
-        let mappingchange, blockingchange;
+        let mappingchange, blockingchange, settingchange;
 
         haketilodb.track.mapping = function (cb) {
             mappingchange = cb;
@@ -99,6 +104,11 @@ def test_pqm_tree_building(driver, execute_in_page):
 
             return [{}, initial_blocking];
         }
+        haketilodb.track.settings = function (cb) {
+            settingchange = cb;
+
+            return [{}, [{name: "default_allow", value: true}]];
+        }
 
         let last_script;
         let unregister_called = 0;
@@ -110,7 +120,7 @@ def test_pqm_tree_building(driver, execute_in_page):
         }
         browser = {contentScripts: {register: register_mock}};
 
-        returnval(start());
+        returnval(start("abracadabra"));
         ''',
         sample_mappings[0:2], sample_blocking[0:2])
 
@@ -125,17 +135,24 @@ def test_pqm_tree_building(driver, execute_in_page):
         dict([('~allow', 1),
               *[(f'inject-{fruit}', {'identifier': f'{fruit}-{best_pattern}'})
                 for fruit in ('banana', 'orange')]])
-    assert tree == extract_tree_data(content_script)
+    cs_values = get_content_script_values(driver, content_script)
+    assert cs_values['haketilo_secret']        == 'abracadabra'
+    assert cs_values['haketilo_pattern_tree']  == tree
+    assert cs_values['haketilo_default_allow'] == True
     assert deregistrations == 0
 
     def condition_all_added(driver):
         last_script = execute_in_page('returnval(last_script);')
+        cs_values = get_content_script_values(driver, last_script)
         nums = [i for i in range(len(sample_blocking)) if i > 1]
-        return (all([('gotmyown%sdoma' % i) in last_script for i in nums]) and
+        return (cs_values['haketilo_default_allow'] == False and
+                all([('gotmyown%sdoma' % i) in last_script for i in nums]) and
                 all([m['identifier'] in last_script for m in sample_mappings]))
 
     execute_in_page(
         '''
+        const new_setting_val = {name: "default_allow", value: false};
+        settingchange({key: "default_allow", new_val: new_setting_val});
         for (const mapping of arguments[0])
             mappingchange({key: mapping.identifier, new_val: mapping});
         for (const blocking of arguments[1])
@@ -163,7 +180,8 @@ def test_pqm_tree_building(driver, execute_in_page):
 
     def condition_all_removed(driver):
         content_script = execute_in_page('returnval(last_script);')
-        return extract_tree_data(content_script) == {}
+        cs_values = get_content_script_values(driver, content_script)
+        return cs_values['haketilo_pattern_tree'] == {}
 
     execute_in_page(
         '''
diff --git a/test/unit/test_popup.py b/test/unit/test_popup.py
index 5319d72..bc53e6c 100644
--- a/test/unit/test_popup.py
+++ b/test/unit/test_popup.py
@@ -62,6 +62,10 @@ mocked_page_infos = {
         **unprivileged_page_info,
         'mapping': 'm1',
         'payload': {'identifier': 'res1'}
+    },
+    'error': {
+        **unprivileged_page_info,
+        'error': True
     }
 }
 
@@ -143,7 +147,7 @@ def test_popup_display(driver, execute_in_page, page_info_key):
     assert by_id['page_url'].text == mocked_page_infos[page_info_key]['url']
     assert not by_id['repo_query_container'].is_displayed()
 
-    if 'blocked' in page_info_key or page_info_key == 'mapping':
+    if 'blocked' in page_info_key or page_info_key in ('mapping', 'error'):
         assert by_id['scripts_blocked'].text.lower() == 'yes'
     elif 'allowed' in page_info_key:
         assert by_id['scripts_blocked'].text.lower() == 'no'
@@ -167,6 +171,9 @@ def test_popup_display(driver, execute_in_page, page_info_key):
     elif 'default' in page_info_key:
         'by default_policy)' in mapping_text
 
+    if page_info_key == 'error':
+        assert mapping_text == 'None (error occured when determining policy)'
+
 @pytest.mark.ext_data(popup_ext_data)
 @pytest.mark.usefixtures('webextension')
 def test_popup_repo_query(driver, execute_in_page):
diff --git a/test/unit/test_webrequest.py b/test/unit/test_webrequest.py
index 598f43b..fb24b3d 100644
--- a/test/unit/test_webrequest.py
+++ b/test/unit/test_webrequest.py
@@ -30,6 +30,8 @@ def webrequest_js():
             ''';
             // Mock pattern tree.
             tree = pqt.make();
+            // Mock default allow.
+            default_allow = {name: "default_allow", value: true};
 
             // Rule to block scripts.
             pqt.register(tree, "https://site.with.scripts.block.ed/***",
@@ -40,10 +42,6 @@ def webrequest_js():
             pqt.register(tree, "https://site.with.paylo.ad/***",
                          "somemapping", {identifier: "someresource"});
 
-            // Mock IndexedDB.
-            haketilodb.track.settings =
-                () => [{}, [{name: "default_allow", value: true}]];
-
             // Mock stream_filter.
             stream_filter.apply = (details, headers, policy) => headers;