utilize Pattern Tree to decide the policy to use and modify HTTP response headers according to that policy

This commit also enhances the build script so that preprocessor conditionals can now use operators '&&' and '||'.
The features being developed are not yet included in the actual Haketilo build.
Some of the new source files contain similar functionality to other ones already existing in the source tree. At some point the latter will be removed.

9 files changed, 481 insertions, 153 deletions

diff --git a/test/extension_crafting.py b/test/extension_crafting.py
index 9b985b3..df45d26 100644
--- a/test/extension_crafting.py
+++ b/test/extension_crafting.py
@@ -58,6 +58,7 @@ def manifest_template():
 	    '<all_urls>',
 	    'unlimitedStorage'
         ],
+        'content_security_policy': "default-src 'self'; script-src 'self' https://serve.scrip.ts;",
         'web_accessible_resources': ['testpage.html'],
         'background': {
 	    'persistent': True,
diff --git a/test/profiles.py b/test/profiles.py
index 795a0db..acdecb6 100755
--- a/test/profiles.py
+++ b/test/profiles.py
@@ -34,22 +34,9 @@ from .misc_constants import *
 
 class HaketiloFirefox(webdriver.Firefox):
     """
-    This wrapper class around selenium.webdriver.Firefox adds a `loaded_scripts`
-    instance property that gets resetted to an empty array every time the
-    `get()` method is called and also facilitates removing the temporary
-    profile directory after Firefox quits.
+    This wrapper class around selenium.webdriver.Firefox facilitates removing
+    the temporary profile directory after Firefox quits.
     """
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.reset_loaded_scripts()
-
-    def reset_loaded_scripts(self):
-        self.loaded_scripts = []
-
-    def get(self, *args, **kwargs):
-        self.reset_loaded_scripts()
-        super().get(*args, **kwargs)
-
     def quit(self, *args, **kwargs):
         profile_path = self.firefox_profile.path
         super().quit(*args, **kwargs)
@@ -71,8 +58,13 @@ def set_profile_proxy(profile, proxy_host, proxy_port):
         profile.set_preference(f'network.proxy.backup.{proto}',      '')
         profile.set_preference(f'network.proxy.backup.{proto}_port', 0)
 
-def set_profile_console_logging(profile):
-    profile.set_preference('devtools.console.stdout.content', True)
+def set_profile_csp_enabled(profile):
+    """
+    By default, Firefox Driver disables CSP. The extension we're testing uses
+    CSP extensively, so we use this function to prepare a Firefox profile that
+    has it enabled.
+    """
+    profile.set_preference('security.csp.enable', True)
 
 # The function below seems not to work for extensions that are
 # temporarily-installed in Firefox safe mode. Testing is needed to see if it
@@ -97,7 +89,7 @@ def firefox_safe_mode(firefox_binary=default_firefox_binary,
     """
     profile = webdriver.FirefoxProfile()
     set_profile_proxy(profile, proxy_host, proxy_port)
-    set_profile_console_logging(profile)
+    set_profile_csp_enabled(profile)
 
     options = Options()
     options.add_argument('--safe-mode')
@@ -117,7 +109,7 @@ def firefox_with_profile(firefox_binary=default_firefox_binary,
     """
     profile = webdriver.FirefoxProfile(profile_dir)
     set_profile_proxy(profile, proxy_host, proxy_port)
-    set_profile_console_logging(profile)
+    set_profile_csp_enabled(profile)
     set_webextension_uuid(profile, default_haketilo_id)
 
     return HaketiloFirefox(firefox_profile=profile,
diff --git a/test/script_loader.py b/test/script_loader.py
index f66f9ae..53de779 100644
--- a/test/script_loader.py
+++ b/test/script_loader.py
@@ -65,7 +65,7 @@ def load_script(path, code_to_add=None):
 
     awk = subprocess.run(['awk', '-f', str(awk_script), '--', '-D', 'MOZILLA',
                           '-D', 'MV2', '-D', 'TEST', '-D', 'UNIT_TEST',
-                          '--output=amalgamate-js:' + key],
+                          '-D', 'DEBUG', '--output=amalgamate-js:' + key],
                          stdout=subprocess.PIPE, cwd=script_root, check=True)
     script = awk.stdout.decode()
     script_cache[key] = script
diff --git a/test/unit/conftest.py b/test/unit/conftest.py
index f9a17f8..beffaf5 100644
--- a/test/unit/conftest.py
+++ b/test/unit/conftest.py
@@ -34,6 +34,7 @@ from selenium.webdriver.support import expected_conditions as EC
 from ..profiles           import firefox_safe_mode
 from ..server             import do_an_internet
 from ..extension_crafting import make_extension
+from ..world_wide_library import start_serving_script, dump_scripts
 
 @pytest.fixture(scope="package")
 def proxy():
@@ -77,55 +78,55 @@ def webextension(driver, request):
     driver.uninstall_addon(addon_id)
     ext_path.unlink()
 
-script_injecting_script = '''\
+script_injector_script = '''\
 /*
  * Selenium by default executes scripts in some weird one-time context. We want
  * separately-loaded scripts to be able to access global variables defined
  * before, including those declared with `const` or `let`. To achieve that, we
- * run our scripts by injecting them into the page inside a <script> tag. We use
- * custom properties of the `window` object to communicate with injected code.
+ * run our scripts by injecting them into the page with a <script> tag that runs
+ * javascript served by our proxy. We use custom properties of the `window`
+ * object to communicate with injected code.
  */
-
-const script_elem = document.createElement('script');
-script_elem.textContent = arguments[0];
-
-delete window.haketilo_selenium_return_value;
-delete window.haketilo_selenium_exception;
-window.returnval = (val => window.haketilo_selenium_return_value = val);
-window.arguments = arguments[1];
-
-document.body.append(script_elem);
-
-/*
- * To ease debugging, we want this script to signal all exceptions from the
- * injectee.
- */
-try {
+const inject = async () => {
+    delete window.haketilo_selenium_return_value;
+    delete window.haketilo_selenium_exception;
+    window.returnval = val => window.haketilo_selenium_return_value = val;
+
+    const injectee = document.createElement('script');
+    injectee.src = arguments[0];
+    injectee.type = "application/javascript";
+    injectee.async = true;
+    const prom = new Promise(cb => injectee.onload = cb);
+
+    window.arguments = arguments[1];
+    document.body.append(injectee);
+
+    await prom;
+
+    /*
+     * To ease debugging, we want this script to signal all exceptions from the
+     * injectee.
+     */
     if (window.haketilo_selenium_exception !== false)
-        throw 'Error in injected script! Check your geckodriver.log!';
-} finally {
-    script_elem.remove();
-}
+        throw ['haketilo_selenium_error',
+               'Error in injected script! Check your geckodriver.log and ./injected_scripts/!'];
 
-return window.haketilo_selenium_return_value;
+    return window.haketilo_selenium_return_value;
+}
+return inject();
 '''
 
 def _execute_in_page_context(driver, script, args):
     script = script + '\n;\nwindow.haketilo_selenium_exception = false;'
-    driver.loaded_scripts.append(script)
+    script_url = start_serving_script(script)
+
     try:
-        return driver.execute_script(script_injecting_script, script, args)
+        result = driver.execute_script(script_injector_script, script_url, args)
+        if type(result) == list and result[0] == 'haketilo_selenium_error':
+            raise Exception(result[1])
+        return result
     except Exception as e:
-        import sys
-
-        print("Scripts loaded since driver's last get() method call:",
-              file=sys.stderr)
-
-        for script in driver.loaded_scripts:
-            lines = enumerate(script.split('\n'), 1)
-            for err_info in [('===',), *lines]:
-                print(*err_info, file=sys.stderr)
-
+        dump_scripts()
         raise e from None
 
 # Some fixtures here just define functions that operate on driver. We should
diff --git a/test/unit/test_indexeddb.py b/test/unit/test_indexeddb.py
index 476690c..df3df81 100644
--- a/test/unit/test_indexeddb.py
+++ b/test/unit/test_indexeddb.py
@@ -75,26 +75,9 @@ def make_sample_mapping():
 def file_ref(file_name):
     return {'file': file_name, 'hash_key': sample_files[file_name]['hash_key']}
 
-@pytest.mark.get_page('https://gotmyowndoma.in')
-def test_haketilodb_save_remove(execute_in_page):
-    """
-    indexeddb.js facilitates operating on Haketilo's internal database.
-    Verify database operations work properly.
-    """
-    execute_in_page(indexeddb_js())
-    # Mock some unwanted imports.
+def clear_indexeddb(execute_in_page):
     execute_in_page(
         '''{
-        const broadcast_mock = {};
-        const nop = () => {};
-        for (const key in broadcast)
-            broadcast_mock[key] = nop;
-        broadcast = broadcast_mock;
-        }''')
-
-    # Start with no database.
-    execute_in_page(
-        '''
         async function delete_db() {
             if (db) {
                 db.close();
@@ -108,12 +91,13 @@ def test_haketilodb_save_remove(execute_in_page):
         }
 
         returnval(delete_db());
-        '''
+        }'''
     )
 
+def get_db_contents(execute_in_page):
     # Facilitate retrieving all IndexedDB contents.
-    execute_in_page(
-        '''
+    return execute_in_page(
+        '''{
         async function get_database_contents()
         {
             const db = await get_db();
@@ -130,20 +114,45 @@ def test_haketilodb_save_remove(execute_in_page):
             store_names_reqs.forEach(([sn, req]) => result[sn] = req.result);
             return result;
         }
-        ''')
+        returnval(get_database_contents());
+        }''')
+
+def mock_broadcast(execute_in_page):
+    execute_in_page(
+        '''{
+        const broadcast_mock = {};
+        const nop = () => {};
+        for (const key in broadcast)
+            broadcast_mock[key] = nop;
+        broadcast = broadcast_mock;
+        }''')
+
+@pytest.mark.get_page('https://gotmyowndoma.in')
+def test_haketilodb_item_modifications(driver, execute_in_page):
+    """
+    indexeddb.js facilitates operating on Haketilo's internal database.
+    Verify database operations on mappings/resources work properly.
+    """
+    execute_in_page(indexeddb_js())
+    mock_broadcast(execute_in_page)
+
+    # Start with no database.
+    clear_indexeddb(execute_in_page)
 
     sample_item = make_sample_resource()
     sample_item['source_copyright'][0]['extra_prop'] = True
 
-    database_contents = execute_in_page(
+    execute_in_page(
         '''{
         const promise = start_items_transaction(["resources"], arguments[1])
             .then(ctx => save_item(arguments[0], ctx).then(() => ctx))
-            .then(finalize_items_transaction)
-            .then(get_database_contents);
+            .then(finalize_transaction);
         returnval(promise);
         }''',
         sample_item, sample_files_by_hash)
+
+    database_contents = get_db_contents(execute_in_page)
+
     assert len(database_contents['files']) == 4
     assert all([sample_files_by_hash[file['hash_key']] == file['contents']
                 for file in database_contents['files']])
@@ -162,31 +171,33 @@ def test_haketilodb_save_remove(execute_in_page):
     sample_item['scripts'].append(file_ref('combined.js'))
     incomplete_files = {**sample_files_by_hash}
     incomplete_files.pop(sample_files['combined.js']['hash_key'])
-    result = execute_in_page(
+    exception = execute_in_page(
         '''{
-        const promise = (async () => {
+        const args = arguments;
+        async function try_add_item()
+        {
             const context =
-                await start_items_transaction(["resources"], arguments[1]);
+                await start_items_transaction(["resources"], args[1]);
             try {
-                await save_item(arguments[0], context);
-                await finalize_items_transaction(context);
-                return {};
+                await save_item(args[0], context);
+                await finalize_transaction(context);
+                return;
             } catch(e) {
-                var exception = e;
+                return e;
             }
-
-            return {exception, db_contents: await get_database_contents()};
-        })();
-        returnval(promise);
+        }
+        returnval(try_add_item());
         }''',
         sample_item, incomplete_files)
 
-    assert result
-    assert 'file not present' in result['exception']
+    previous_database_contents = database_contents
+    database_contents = get_db_contents(execute_in_page)
+
+    assert 'file not present' in exception
     for key, val in database_contents.items():
         keyfun = lambda item: item.get('hash_key') or item['identifier']
-        assert sorted(result['db_contents'][key], key=keyfun) \
-            == sorted(val,                        key=keyfun)
+        assert sorted(previous_database_contents[key], key=keyfun) \
+            == sorted(val,                             key=keyfun)
 
     # See if adding another item that partially uses first's files works OK.
     sample_item = make_sample_mapping()
@@ -194,12 +205,13 @@ def test_haketilodb_save_remove(execute_in_page):
         '''{
         const promise = start_items_transaction(["mappings"], arguments[1])
             .then(ctx => save_item(arguments[0], ctx).then(() => ctx))
-            .then(finalize_items_transaction)
-            .then(get_database_contents);
+            .then(finalize_transaction);
         returnval(promise);
         }''',
         sample_item, sample_files_by_hash)
 
+    database_contents = get_db_contents(execute_in_page)
+
     names = ['README.md', 'report.spdx', 'LICENSES/somelicense.txt', 'hello.js',
              'bye.js']
     sample_files_list = [sample_files[name] for name in names]
@@ -222,17 +234,18 @@ def test_haketilodb_save_remove(execute_in_page):
     # Try removing the items to get an empty database again.
     results = [None, None]
     for i, item_type in enumerate(['resource', 'mapping']):
-         results[i] = execute_in_page(
+         execute_in_page(
             f'''{{
             const remover = remove_{item_type};
             const promise =
                 start_items_transaction(["{item_type}s"], {{}})
                 .then(ctx => remover('helloapple', ctx).then(() => ctx))
-                .then(finalize_items_transaction)
-                .then(get_database_contents);
+                .then(finalize_transaction);
             returnval(promise);
             }}''')
 
+         results[i] = get_db_contents(execute_in_page)
+
     names = ['README.md', 'report.spdx']
     sample_files_list = [sample_files[name] for name in names]
     uses_list = [1, 1]
@@ -271,22 +284,48 @@ def test_haketilodb_save_remove(execute_in_page):
         },
         'files': sample_files_by_hash
     }
-    database_contents = execute_in_page(
-        '''
-        initial_data = arguments[0];
-        returnval(delete_db().then(() => get_database_contents()));
-        ''',
-        initial_data)
+
+    clear_indexeddb(execute_in_page)
+    execute_in_page('initial_data = arguments[0];', initial_data)
+    database_contents = get_db_contents(execute_in_page)
+
     assert database_contents['resources'] == [sample_resource]
     assert database_contents['mappings']  == [sample_mapping]
 
+@pytest.mark.get_page('https://gotmyowndoma.in')
+def test_haketilodb_settings(driver, execute_in_page):
+    """
+    indexeddb.js facilitates operating on Haketilo's internal database.
+    Verify database assigning/retrieving values of simple "settings" works
+    properly.
+    """
+    execute_in_page(indexeddb_js())
+    mock_broadcast(execute_in_page)
+
+    # Start with no database.
+    clear_indexeddb(execute_in_page)
+
+    assert get_db_contents(execute_in_page)['settings'] == []
+
+    assert execute_in_page('returnval(get_setting("option15"));') == None
+
+    execute_in_page('returnval(set_setting("option15", "disable"));')
+    assert execute_in_page('returnval(get_setting("option15"));') == 'disable'
+
+    execute_in_page('returnval(set_setting("option15", "enable"));')
+    assert execute_in_page('returnval(get_setting("option15"));') == 'enable'
+
 test_page_html = '''
 <!DOCTYPE html>
 <script src="/testpage.js"></script>
+<script>console.log("inline!")</script>
+<script nonce="123456789">console.log("inline nonce!")</script>
 <h2>resources</h2>
 <ul id="resources"></ul>
 <h2>mappings</h2>
 <ul id="mappings"></ul>
+<h2>settings</h2>
+<ul id="settings"></ul>
 '''
 
 @pytest.mark.ext_data({
@@ -328,15 +367,21 @@ def test_haketilodb_track(driver, execute_in_page, wait_elem_text):
     }
     for window in reversed(windows):
         driver.switch_to.window(window)
-        execute_in_page('initial_data = arguments[0];', initial_data)
-
-    # See if track_*() functions properly return the already-existing items.
+        try :
+            driver.execute_script('console.log("uuuuuuu");')
+            execute_in_page('initial_data = arguments[0];', initial_data)
+        except:
+            from time import sleep
+            sleep(100000)
+        execute_in_page('returnval(set_setting("option15", "123"));')
+
+    # See if track.*() functions properly return the already-existing items.
     execute_in_page(
         '''
         function update_item(store_name, change)
         {
             console.log('update', ...arguments);
-            const elem_id = `${store_name}_${change.identifier}`;
+            const elem_id = `${store_name}_${change.key}`;
             let elem = document.getElementById(elem_id);
             elem = elem || document.createElement("li");
             elem.id = elem_id;
@@ -348,35 +393,32 @@ def test_haketilodb_track(driver, execute_in_page, wait_elem_text):
 
         let resource_tracking, resource_items, mapping_tracking, mapping_items;
 
-        async function start_tracking()
+        async function start_reporting()
         {
-            const update_resource = change => update_item("resources", change);
-            const update_mapping  = change => update_item("mappings",  change);
-
-            [resource_tracking, resource_items] =
-                await track_resources(update_resource);
-            [mapping_tracking, mapping_items] =
-                await track_mappings(update_mapping);
-
-            for (const item of resource_items)
-                update_resource({identifier: item.identifier, new_val: item});
-            for (const item of mapping_items)
-                update_mapping({identifier: item.identifier, new_val: item});
+            for (const store_name of ["resources", "mappings", "settings"]) {
+                [tracking, items] =
+                    await track[store_name](ch => update_item(store_name, ch));
+                const prop = store_name === "settings" ? "name" : "identifier";
+                for (const item of items)
+                    update_item(store_name, {key: item[prop], new_val: item});
+            }
         }
 
-        returnval(start_tracking());
+        returnval(start_reporting());
         ''')
 
     item_counts = driver.execute_script(
         '''
         const childcount = id => document.getElementById(id).childElementCount;
-        return ["resources", "mappings"].map(childcount);
+        return ["resources", "mappings", "settings"].map(childcount);
         ''')
-    assert item_counts == [1, 1]
+    assert item_counts == [1, 1, 1]
     resource_json = driver.find_element_by_id('resources_helloapple').text
     mapping_json =  driver.find_element_by_id('mappings_helloapple').text
+    setting_json =  driver.find_element_by_id('settings_option15').text
     assert json.loads(resource_json) == sample_resource
     assert json.loads(mapping_json)  == sample_mapping
+    assert json.loads(setting_json)  == {'name': 'option15', 'value': '123'}
 
     # See if item additions get tracked properly.
     driver.switch_to.window(windows[1])
@@ -398,14 +440,17 @@ def test_haketilodb_track(driver, execute_in_page, wait_elem_text):
         'files': sample_files_by_hash
     }
     execute_in_page('returnval(save_items(arguments[0]));', sample_data)
+    execute_in_page('returnval(set_setting("option22", "abc"));')
 
     driver.switch_to.window(windows[0])
     driver.implicitly_wait(10)
     resource_json = driver.find_element_by_id('resources_helloapple-copy').text
     mapping_json =  driver.find_element_by_id('mappings_helloapple-copy').text
+    setting_json =  driver.find_element_by_id('settings_option22').text
     driver.implicitly_wait(0)
     assert json.loads(resource_json) == sample_resource2
     assert json.loads(mapping_json)  == sample_mapping2
+    assert json.loads(setting_json)  == {'name': 'option22', 'value': 'abc'}
 
     # See if item deletions get tracked properly.
     driver.switch_to.window(windows[1])
@@ -417,7 +462,8 @@ def test_haketilodb_track(driver, execute_in_page, wait_elem_text):
             const ctx = await start_items_transaction(store_names, {});
             await remove_resource("helloapple", ctx);
             await remove_mapping("helloapple-copy", ctx);
-            await finalize_items_transaction(ctx);
+            await finalize_transaction(ctx);
+            await set_setting("option22", null);
         }
         returnval(remove_items());
         }''')
@@ -430,7 +476,8 @@ def test_haketilodb_track(driver, execute_in_page, wait_elem_text):
                 return False
             except WebDriverException:
                 pass
-        return True
+        option_text = driver.find_element_by_id('settings_option22').text
+        return json.loads(option_text)['value'] == None
 
     driver.switch_to.window(windows[0])
     WebDriverWait(driver, 10).until(condition_items_absent)
diff --git a/test/unit/test_patterns_query_manager.py b/test/unit/test_patterns_query_manager.py
index 8ae7c28..ae1f490 100644
--- a/test/unit/test_patterns_query_manager.py
+++ b/test/unit/test_patterns_query_manager.py
@@ -25,10 +25,9 @@ from selenium.webdriver.support.ui import WebDriverWait
 from ..script_loader import load_script
 
 def simple_sample_mapping(patterns, fruit):
-    if type(patterns) is list:
-        payloads = dict([(p, {'identifier': fruit}) for p in patterns])
-    else:
-        payloads = {patterns: {'identifier': fruit}}
+    if type(patterns) is not list:
+        patterns = [patterns]
+    payloads = dict([(p, {'identifier': f'{fruit}-{p}'}) for p in patterns])
     return {
         'source_copyright': [],
         'type': 'mapping',
@@ -36,9 +35,13 @@ def simple_sample_mapping(patterns, fruit):
         'payloads': payloads
     }
 
-content_script_re = re.compile(r'this.haketilo_pattern_tree = (.*);')
+content_script_tree_re = re.compile(r'this.haketilo_pattern_tree = (.*);')
 def extract_tree_data(content_script_text):
-    return json.loads(content_script_re.search(content_script_text)[1])
+    return json.loads(content_script_tree_re.search(content_script_text)[1])
+
+content_script_mapping_re = re.compile(r'this.haketilo_mappings = (.*);')
+def extract_mappings_data(content_script_text):
+    return json.loads(content_script_mapping_re.search(content_script_text)[1])
 
 # Fields that are not relevant for testing are omitted from these mapping
 # definitions.
@@ -82,7 +85,7 @@ def test_pqm_tree_building(driver, execute_in_page):
 
             return [{}, initial_mappings];
         }
-        haketilodb.track_mappings = track_mock;
+        haketilodb.track.mappings = track_mock;
 
         let last_script;
         let unregister_called = 0;
@@ -104,7 +107,10 @@ def test_pqm_tree_building(driver, execute_in_page):
                    tree, last_script, unregister_called]);
         ''',
         'https://gotmyowndoma.in/index.html')
-    assert found == dict([(m['identifier'], m) for m in sample_mappings[0:2]])
+    best_pattern = 'https://gotmyowndoma.in/index.html'
+    assert found == \
+        dict([(f'inject-{fruit}', {'identifier': f'{fruit}-{best_pattern}'})
+              for fruit in ('banana', 'orange')])
     assert tree == extract_tree_data(content_script)
     assert deregistrations == 0
 
@@ -114,12 +120,8 @@ def test_pqm_tree_building(driver, execute_in_page):
 
     execute_in_page(
         '''
-        for (const mapping of arguments[0]) {
-            mappingchange({
-                identifier: mapping.identifier,
-                new_val: mapping
-            });
-        }
+        for (const mapping of arguments[0])
+            mappingchange({key: mapping.identifier, new_val: mapping});
         ''',
         sample_mappings[2:])
     WebDriverWait(driver, 10).until(condition_mappings_added)
@@ -129,7 +131,8 @@ def test_pqm_tree_building(driver, execute_in_page):
 
     def condition_odd_removed(driver):
         last_script = execute_in_page('returnval(last_script);')
-        return all([id not in last_script for id in odd])
+        return (all([id not in last_script for id in odd]) and
+                all([id in last_script for id in even]))
 
     def condition_all_removed(driver):
         content_script = execute_in_page('returnval(last_script);')
@@ -137,7 +140,7 @@ def test_pqm_tree_building(driver, execute_in_page):
 
     execute_in_page(
         '''
-        arguments[0].forEach(identifier => mappingchange({identifier}));
+        arguments[0].forEach(identifier => mappingchange({key: identifier}));
         ''',
         odd)
 
@@ -145,7 +148,7 @@ def test_pqm_tree_building(driver, execute_in_page):
 
     execute_in_page(
         '''
-        arguments[0].forEach(identifier => mappingchange({identifier}));
+        arguments[0].forEach(identifier => mappingchange({key: identifier}));
         ''',
         even)
 
@@ -224,7 +227,7 @@ def test_pqm_script_injection(driver, execute_in_page):
             const ctx = await start_items_transaction(["mappings"], {});
             for (const id of identifiers)
                 await remove_mapping(id, ctx);
-            await finalize_items_transaction(ctx);
+            await finalize_transaction(ctx);
         }
         returnval(remove_items());
         }''',
diff --git a/test/unit/test_policy_deciding.py b/test/unit/test_policy_deciding.py
new file mode 100644
index 0000000..a360537
--- /dev/null
+++ b/test/unit/test_policy_deciding.py
@@ -0,0 +1,121 @@
+# SPDX-License-Identifier: CC0-1.0
+
+"""
+Haketilo unit tests - determining what to do on a given web page
+"""
+
+# This file is part of Haketilo
+#
+# Copyright (C) 2021, Wojtek Kosior <koszko@koszko.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the CC0 1.0 Universal License as published by
+# the Creative Commons Corporation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# CC0 1.0 Universal License for more details.
+
+import re
+from hashlib import sha256
+import pytest
+
+from ..script_loader import load_script
+
+csp_re = re.compile(r'^\S+\s+\S+;(?:\s+\S+\s+\S+;)*$')
+rule_re = re.compile(r'^\s*(?P<src_kind>\S+)\s+(?P<allowed_origins>\S+)$')
+def parse_csp(csp):
+    '''
+    Parsing of CSP string into a dict. A simplified format of CSP is assumed.
+    '''
+    assert csp_re.match(csp)
+
+    result = {}
+
+    for rule in csp.split(';')[:-1]:
+        match = rule_re.match(rule)
+        result[match.group('src_kind')] = match.group('allowed_origins')
+
+    return result
+
+@pytest.mark.get_page('https://gotmyowndoma.in')
+def test_decide_policy(execute_in_page):
+    """
+    policy.js contains code that, using a Pattern Query Tree instance and a URL,
+    decides what Haketilo should do on a page opened at that URL, i.e. whether
+    it should block or allow script execution and whether it should inject its
+    own scripts and which ones. Test that the policy object gets constructed
+    properly.
+    """
+    execute_in_page(load_script('common/policy.js'))
+
+    policy = execute_in_page(
+        '''
+        returnval(decide_policy(pqt.make(), "http://unkno.wn/", true, "abcd"));
+        ''')
+    assert policy['allow'] == True
+    for prop in ('mapping', 'payload', 'nonce', 'csp'):
+        assert prop not in policy
+
+    policy = execute_in_page(
+        '''{
+        const tree = pqt.make();
+        pqt.register(tree, "http://kno.wn", "allowed", {allow: true});
+        returnval(decide_policy(tree, "http://kno.wn/", false, "abcd"));
+        }''')
+    assert policy['allow'] == True
+    assert policy['mapping'] == 'allowed'
+    for prop in ('payload', 'nonce', 'csp'):
+        assert prop not in policy
+
+    policy = execute_in_page(
+        '''
+        returnval(decide_policy(pqt.make(), "http://unkno.wn/", false, "abcd"));
+        '''
+        )
+    assert policy['allow'] == False
+    for prop in ('mapping', 'payload', 'nonce'):
+        assert prop not in policy
+    assert parse_csp(policy['csp']) == {
+        'prefetch-src':    "'none'",
+        'script-src-attr': "'none'",
+        'script-src':      "'none'",
+        'script-src-elem': "'none'"
+    }
+
+    policy = execute_in_page(
+        '''{
+        const tree = pqt.make();
+        pqt.register(tree, "http://kno.wn", "disallowed", {allow: false});
+        returnval(decide_policy(tree, "http://kno.wn/", true, "abcd"));
+        }''')
+    assert policy['allow'] == False
+    assert policy['mapping'] == 'disallowed'
+    for prop in ('payload', 'nonce'):
+        assert prop not in policy
+    assert parse_csp(policy['csp']) == {
+        'prefetch-src':    "'none'",
+        'script-src-attr': "'none'",
+        'script-src':      "'none'",
+        'script-src-elem': "'none'"
+    }
+
+    policy = execute_in_page(
+        '''{
+        const tree = pqt.make();
+        pqt.register(tree, "http://kno.wn", "m1", {identifier: "res1"});
+        returnval(decide_policy(tree, "http://kno.wn/", true, "abcd"));
+        }''')
+    assert policy['allow'] == False
+    assert policy['mapping'] == 'm1'
+    assert policy['payload'] == {'identifier': 'res1'}
+
+    assert policy['nonce'] == \
+        sha256('m1:res1:http://kno.wn/:abcd'.encode()).digest().hex()
+    assert parse_csp(policy['csp']) == {
+        'prefetch-src':    f"'none'",
+        'script-src-attr': f"'none'",
+        'script-src':      f"'nonce-{policy['nonce']}'",
+        'script-src-elem': f"'nonce-{policy['nonce']}'"
+    }
diff --git a/test/unit/test_webrequest.py b/test/unit/test_webrequest.py
new file mode 100644
index 0000000..6af2758
--- /dev/null
+++ b/test/unit/test_webrequest.py
@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: CC0-1.0
+
+"""
+Haketilo unit tests - modifying requests using webRequest API
+"""
+
+# This file is part of Haketilo
+#
+# Copyright (C) 2021, Wojtek Kosior <koszko@koszko.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the CC0 1.0 Universal License as published by
+# the Creative Commons Corporation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# CC0 1.0 Universal License for more details.
+
+import re
+from hashlib import sha256
+import pytest
+
+from ..script_loader import load_script
+
+def webrequest_js():
+    return (load_script('background/webrequest.js',
+                        '#IMPORT common/patterns_query_tree.js AS pqt') +
+            ''';
+            // Mock pattern tree.
+            tree = pqt.make();
+            pqt.register(tree, "https://site.with.scripts.block.ed/***",
+                         "disallowed", {allow: false});
+            pqt.register(tree, "https://site.with.paylo.ad/***",
+                         "somemapping", {identifier: "someresource"});
+
+            // Mock IndexedDB.
+            haketilodb.track.settings =
+                () => [{}, [{name: "default_allow", value: true}]];
+
+            // Mock stream_filter.
+            stream_filter.apply = (details, headers, policy) => headers;
+
+            // Mock secret and start webrequest operations.
+            start("somesecret");
+            ''')
+
+def are_scripts_allowed(driver, nonce=None):
+        return driver.execute_script(
+            '''
+            document.scripts_allowed = false;
+            const script = document.createElement("script");
+            script.innerHTML = "document.scripts_allowed = true;";
+            if (arguments[0])
+                script.setAttribute("nonce", arguments[0]);
+            document.head.append(script);
+            return document.scripts_allowed;
+            ''',
+            nonce)
+
+@pytest.mark.ext_data({'background_script': webrequest_js})
+@pytest.mark.usefixtures('webextension')
+def test_on_headers_received(driver, execute_in_page):
+    for attempt in range(10):
+        driver.get('https://site.with.scripts.block.ed/')
+
+        if not are_scripts_allowed(driver):
+            break
+        assert attempt != 9
+
+    driver.get('https://site.with.scripts.allow.ed/')
+    assert are_scripts_allowed(driver)
+
+    driver.get('https://site.with.paylo.ad/')
+    assert not are_scripts_allowed(driver)
+    source = 'somemapping:someresource:https://site.with.paylo.ad/index.html:somesecret'
+    assert are_scripts_allowed(driver, sha256(source.encode()).digest().hex())
diff --git a/test/world_wide_library.py b/test/world_wide_library.py
index 860c987..43d3512 100644
--- a/test/world_wide_library.py
+++ b/test/world_wide_library.py
@@ -27,13 +27,99 @@ Our helpful little stand-in for the Internet
 # file's license. Although I request that you do not make use this code
 # in a proprietary program, I am not going to enforce this in court.
 
+from hashlib import sha256
+from pathlib import Path
+from shutil import rmtree
+from threading import Lock
+
 from .misc_constants import here
 
+served_scripts = {}
+served_scripts_lock = Lock()
+
+def start_serving_script(script_text):
+    """
+    Register given script so that it is served at
+    https://serve.scrip.ts/?sha256=<script's_sha256_sum>
+
+    Returns the URL at which script will be served.
+
+    This function lacks thread safety. Might moght consider fixing this if it
+    turns
+    """
+    sha256sum = sha256(script_text.encode()).digest().hex()
+    served_scripts_lock.acquire()
+    served_scripts[sha256sum] = script_text
+    served_scripts_lock.release()
+
+    return f'https://serve.scrip.ts/?sha256={sha256sum}'
+
+def serve_script(command, get_params, post_params):
+    """
+    info() callback to pass to request-handling code in server.py. Facilitates
+    serving scripts that have been registered with start_serving_script().
+    """
+    served_scripts_lock.acquire()
+    try:
+        script = served_scripts.get(get_params['sha256'][0])
+    finally:
+        served_scripts_lock.release()
+    if script is None:
+        return 404, {}, b''
+
+    return 200, {'Content-Type': 'application/javascript'}, script
+
+def dump_scripts(directory='./injected_scripts'):
+    """
+    Write all scripts that have been registered with start_serving_script()
+    under the provided directory. If the directory already exists, it is wiped
+    beforehand. If it doesn't exist, it is created.
+    """
+    directory = Path(directory)
+    rmtree(directory, ignore_errors=True)
+    directory.mkdir(parents=True)
+
+    served_scripts_lock.acquire()
+    for sha256, script in served_scripts.items():
+        with open(directory / sha256, 'wt') as file:
+            file.write(script)
+    served_scripts_lock.release()
+
 catalog = {
-	'http://gotmyowndoma.in': (302, {'location': 'http://gotmyowndoma.in/index.html'}, None),
-	'http://gotmyowndoma.in/': (302, {'location': 'http://gotmyowndoma.in/index.html'}, None),
-	'http://gotmyowndoma.in/index.html': (200, {}, here / 'data' / 'pages' / 'gotmyowndomain.html'),
-	'https://gotmyowndoma.in': (302, {'location': 'https://gotmyowndoma.in/index.html'}, None),
-	'https://gotmyowndoma.in/': (302, {'location': 'https://gotmyowndoma.in/index.html'}, None),
-	'https://gotmyowndoma.in/index.html': (200, {}, here / 'data' / 'pages' / 'gotmyowndomain_https.html')
+    'http://gotmyowndoma.in':
+    (302, {'location': 'http://gotmyowndoma.in/index.html'}, None),
+    'http://gotmyowndoma.in/':
+    (302, {'location': 'http://gotmyowndoma.in/index.html'}, None),
+    'http://gotmyowndoma.in/index.html':
+    (200, {}, here / 'data' / 'pages' / 'gotmyowndomain.html'),
+
+    'https://gotmyowndoma.in':
+    (302, {'location': 'https://gotmyowndoma.in/index.html'}, None),
+    'https://gotmyowndoma.in/':
+    (302, {'location': 'https://gotmyowndoma.in/index.html'}, None),
+    'https://gotmyowndoma.in/index.html':
+    (200, {}, here / 'data' / 'pages' / 'gotmyowndomain_https.html'),
+
+    'https://serve.scrip.ts/': serve_script,
+
+    'https://site.with.scripts.block.ed':
+    (302, {'location': 'https://site.with.scripts.block.ed/index.html'}, None),
+    'https://site.with.scripts.block.ed/':
+    (302, {'location': 'https://site.with.scripts.block.ed/index.html'}, None),
+    'https://site.with.scripts.block.ed/index.html':
+    (200, {}, here / 'data' / 'pages' / 'gotmyowndomain_https.html'),
+
+    'https://site.with.scripts.allow.ed':
+    (302, {'location': 'https://site.with.scripts.allow.ed/index.html'}, None),
+    'https://site.with.scripts.allow.ed/':
+    (302, {'location': 'https://site.with.scripts.allow.ed/index.html'}, None),
+    'https://site.with.scripts.allow.ed/index.html':
+    (200, {}, here / 'data' / 'pages' / 'gotmyowndomain_https.html'),
+
+    'https://site.with.paylo.ad':
+    (302, {'location': 'https://site.with.paylo.ad/index.html'}, None),
+    'https://site.with.paylo.ad/':
+    (302, {'location': 'https://site.with.paylo.ad/index.html'}, None),
+    'https://site.with.paylo.ad/index.html':
+    (200, {}, here / 'data' / 'pages' / 'gotmyowndomain_https.html')
 }