aboutsummaryrefslogtreecommitdiff
path: root/test
diff options
context:
space:
mode:
Diffstat (limited to 'test')
-rw-r--r--test/extension_crafting.py1
-rwxr-xr-xtest/profiles.py30
-rw-r--r--test/script_loader.py2
-rw-r--r--test/unit/conftest.py73
-rw-r--r--test/unit/test_indexeddb.py193
-rw-r--r--test/unit/test_patterns_query_manager.py39
-rw-r--r--test/unit/test_policy_deciding.py121
-rw-r--r--test/unit/test_webrequest.py77
-rw-r--r--test/world_wide_library.py98
9 files changed, 481 insertions, 153 deletions
diff --git a/test/extension_crafting.py b/test/extension_crafting.py
index 9b985b3..df45d26 100644
--- a/test/extension_crafting.py
+++ b/test/extension_crafting.py
@@ -58,6 +58,7 @@ def manifest_template():
'<all_urls>',
'unlimitedStorage'
],
+ 'content_security_policy': "default-src 'self'; script-src 'self' https://serve.scrip.ts;",
'web_accessible_resources': ['testpage.html'],
'background': {
'persistent': True,
diff --git a/test/profiles.py b/test/profiles.py
index 795a0db..acdecb6 100755
--- a/test/profiles.py
+++ b/test/profiles.py
@@ -34,22 +34,9 @@ from .misc_constants import *
class HaketiloFirefox(webdriver.Firefox):
"""
- This wrapper class around selenium.webdriver.Firefox adds a `loaded_scripts`
- instance property that gets resetted to an empty array every time the
- `get()` method is called and also facilitates removing the temporary
- profile directory after Firefox quits.
+ This wrapper class around selenium.webdriver.Firefox facilitates removing
+ the temporary profile directory after Firefox quits.
"""
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.reset_loaded_scripts()
-
- def reset_loaded_scripts(self):
- self.loaded_scripts = []
-
- def get(self, *args, **kwargs):
- self.reset_loaded_scripts()
- super().get(*args, **kwargs)
-
def quit(self, *args, **kwargs):
profile_path = self.firefox_profile.path
super().quit(*args, **kwargs)
@@ -71,8 +58,13 @@ def set_profile_proxy(profile, proxy_host, proxy_port):
profile.set_preference(f'network.proxy.backup.{proto}', '')
profile.set_preference(f'network.proxy.backup.{proto}_port', 0)
-def set_profile_console_logging(profile):
- profile.set_preference('devtools.console.stdout.content', True)
+def set_profile_csp_enabled(profile):
+ """
+ By default, Firefox Driver disables CSP. The extension we're testing uses
+ CSP extensively, so we use this function to prepare a Firefox profile that
+ has it enabled.
+ """
+ profile.set_preference('security.csp.enable', True)
# The function below seems not to work for extensions that are
# temporarily-installed in Firefox safe mode. Testing is needed to see if it
@@ -97,7 +89,7 @@ def firefox_safe_mode(firefox_binary=default_firefox_binary,
"""
profile = webdriver.FirefoxProfile()
set_profile_proxy(profile, proxy_host, proxy_port)
- set_profile_console_logging(profile)
+ set_profile_csp_enabled(profile)
options = Options()
options.add_argument('--safe-mode')
@@ -117,7 +109,7 @@ def firefox_with_profile(firefox_binary=default_firefox_binary,
"""
profile = webdriver.FirefoxProfile(profile_dir)
set_profile_proxy(profile, proxy_host, proxy_port)
- set_profile_console_logging(profile)
+ set_profile_csp_enabled(profile)
set_webextension_uuid(profile, default_haketilo_id)
return HaketiloFirefox(firefox_profile=profile,
diff --git a/test/script_loader.py b/test/script_loader.py
index f66f9ae..53de779 100644
--- a/test/script_loader.py
+++ b/test/script_loader.py
@@ -65,7 +65,7 @@ def load_script(path, code_to_add=None):
awk = subprocess.run(['awk', '-f', str(awk_script), '--', '-D', 'MOZILLA',
'-D', 'MV2', '-D', 'TEST', '-D', 'UNIT_TEST',
- '--output=amalgamate-js:' + key],
+ '-D', 'DEBUG', '--output=amalgamate-js:' + key],
stdout=subprocess.PIPE, cwd=script_root, check=True)
script = awk.stdout.decode()
script_cache[key] = script
diff --git a/test/unit/conftest.py b/test/unit/conftest.py
index f9a17f8..beffaf5 100644
--- a/test/unit/conftest.py
+++ b/test/unit/conftest.py
@@ -34,6 +34,7 @@ from selenium.webdriver.support import expected_conditions as EC
from ..profiles import firefox_safe_mode
from ..server import do_an_internet
from ..extension_crafting import make_extension
+from ..world_wide_library import start_serving_script, dump_scripts
@pytest.fixture(scope="package")
def proxy():
@@ -77,55 +78,55 @@ def webextension(driver, request):
driver.uninstall_addon(addon_id)
ext_path.unlink()
-script_injecting_script = '''\
+script_injector_script = '''\
/*
* Selenium by default executes scripts in some weird one-time context. We want
* separately-loaded scripts to be able to access global variables defined
* before, including those declared with `const` or `let`. To achieve that, we
- * run our scripts by injecting them into the page inside a <script> tag. We use
- * custom properties of the `window` object to communicate with injected code.
+ * run our scripts by injecting them into the page with a <script> tag that runs
+ * javascript served by our proxy. We use custom properties of the `window`
+ * object to communicate with injected code.
*/
-
-const script_elem = document.createElement('script');
-script_elem.textContent = arguments[0];
-
-delete window.haketilo_selenium_return_value;
-delete window.haketilo_selenium_exception;
-window.returnval = (val => window.haketilo_selenium_return_value = val);
-window.arguments = arguments[1];
-
-document.body.append(script_elem);
-
-/*
- * To ease debugging, we want this script to signal all exceptions from the
- * injectee.
- */
-try {
+const inject = async () => {
+ delete window.haketilo_selenium_return_value;
+ delete window.haketilo_selenium_exception;
+ window.returnval = val => window.haketilo_selenium_return_value = val;
+
+ const injectee = document.createElement('script');
+ injectee.src = arguments[0];
+ injectee.type = "application/javascript";
+ injectee.async = true;
+ const prom = new Promise(cb => injectee.onload = cb);
+
+ window.arguments = arguments[1];
+ document.body.append(injectee);
+
+ await prom;
+
+ /*
+ * To ease debugging, we want this script to signal all exceptions from the
+ * injectee.
+ */
if (window.haketilo_selenium_exception !== false)
- throw 'Error in injected script! Check your geckodriver.log!';
-} finally {
- script_elem.remove();
-}
+ throw ['haketilo_selenium_error',
+ 'Error in injected script! Check your geckodriver.log and ./injected_scripts/!'];
-return window.haketilo_selenium_return_value;
+ return window.haketilo_selenium_return_value;
+}
+return inject();
'''
def _execute_in_page_context(driver, script, args):
script = script + '\n;\nwindow.haketilo_selenium_exception = false;'
- driver.loaded_scripts.append(script)
+ script_url = start_serving_script(script)
+
try:
- return driver.execute_script(script_injecting_script, script, args)
+ result = driver.execute_script(script_injector_script, script_url, args)
+ if type(result) == list and result[0] == 'haketilo_selenium_error':
+ raise Exception(result[1])
+ return result
except Exception as e:
- import sys
-
- print("Scripts loaded since driver's last get() method call:",
- file=sys.stderr)
-
- for script in driver.loaded_scripts:
- lines = enumerate(script.split('\n'), 1)
- for err_info in [('===',), *lines]:
- print(*err_info, file=sys.stderr)
-
+ dump_scripts()
raise e from None
# Some fixtures here just define functions that operate on driver. We should
diff --git a/test/unit/test_indexeddb.py b/test/unit/test_indexeddb.py
index 476690c..df3df81 100644
--- a/test/unit/test_indexeddb.py
+++ b/test/unit/test_indexeddb.py
@@ -75,26 +75,9 @@ def make_sample_mapping():
def file_ref(file_name):
return {'file': file_name, 'hash_key': sample_files[file_name]['hash_key']}
-@pytest.mark.get_page('https://gotmyowndoma.in')
-def test_haketilodb_save_remove(execute_in_page):
- """
- indexeddb.js facilitates operating on Haketilo's internal database.
- Verify database operations work properly.
- """
- execute_in_page(indexeddb_js())
- # Mock some unwanted imports.
+def clear_indexeddb(execute_in_page):
execute_in_page(
'''{
- const broadcast_mock = {};
- const nop = () => {};
- for (const key in broadcast)
- broadcast_mock[key] = nop;
- broadcast = broadcast_mock;
- }''')
-
- # Start with no database.
- execute_in_page(
- '''
async function delete_db() {
if (db) {
db.close();
@@ -108,12 +91,13 @@ def test_haketilodb_save_remove(execute_in_page):
}
returnval(delete_db());
- '''
+ }'''
)
+def get_db_contents(execute_in_page):
# Facilitate retrieving all IndexedDB contents.
- execute_in_page(
- '''
+ return execute_in_page(
+ '''{
async function get_database_contents()
{
const db = await get_db();
@@ -130,20 +114,45 @@ def test_haketilodb_save_remove(execute_in_page):
store_names_reqs.forEach(([sn, req]) => result[sn] = req.result);
return result;
}
- ''')
+ returnval(get_database_contents());
+ }''')
+
+def mock_broadcast(execute_in_page):
+ execute_in_page(
+ '''{
+ const broadcast_mock = {};
+ const nop = () => {};
+ for (const key in broadcast)
+ broadcast_mock[key] = nop;
+ broadcast = broadcast_mock;
+ }''')
+
+@pytest.mark.get_page('https://gotmyowndoma.in')
+def test_haketilodb_item_modifications(driver, execute_in_page):
+ """
+ indexeddb.js facilitates operating on Haketilo's internal database.
+ Verify database operations on mappings/resources work properly.
+ """
+ execute_in_page(indexeddb_js())
+ mock_broadcast(execute_in_page)
+
+ # Start with no database.
+ clear_indexeddb(execute_in_page)
sample_item = make_sample_resource()
sample_item['source_copyright'][0]['extra_prop'] = True
- database_contents = execute_in_page(
+ execute_in_page(
'''{
const promise = start_items_transaction(["resources"], arguments[1])
.then(ctx => save_item(arguments[0], ctx).then(() => ctx))
- .then(finalize_items_transaction)
- .then(get_database_contents);
+ .then(finalize_transaction);
returnval(promise);
}''',
sample_item, sample_files_by_hash)
+
+ database_contents = get_db_contents(execute_in_page)
+
assert len(database_contents['files']) == 4
assert all([sample_files_by_hash[file['hash_key']] == file['contents']
for file in database_contents['files']])
@@ -162,31 +171,33 @@ def test_haketilodb_save_remove(execute_in_page):
sample_item['scripts'].append(file_ref('combined.js'))
incomplete_files = {**sample_files_by_hash}
incomplete_files.pop(sample_files['combined.js']['hash_key'])
- result = execute_in_page(
+ exception = execute_in_page(
'''{
- const promise = (async () => {
+ const args = arguments;
+ async function try_add_item()
+ {
const context =
- await start_items_transaction(["resources"], arguments[1]);
+ await start_items_transaction(["resources"], args[1]);
try {
- await save_item(arguments[0], context);
- await finalize_items_transaction(context);
- return {};
+ await save_item(args[0], context);
+ await finalize_transaction(context);
+ return;
} catch(e) {
- var exception = e;
+ return e;
}
-
- return {exception, db_contents: await get_database_contents()};
- })();
- returnval(promise);
+ }
+ returnval(try_add_item());
}''',
sample_item, incomplete_files)
- assert result
- assert 'file not present' in result['exception']
+ previous_database_contents = database_contents
+ database_contents = get_db_contents(execute_in_page)
+
+ assert 'file not present' in exception
for key, val in database_contents.items():
keyfun = lambda item: item.get('hash_key') or item['identifier']
- assert sorted(result['db_contents'][key], key=keyfun) \
- == sorted(val, key=keyfun)
+ assert sorted(previous_database_contents[key], key=keyfun) \
+ == sorted(val, key=keyfun)
# See if adding another item that partially uses first's files works OK.
sample_item = make_sample_mapping()
@@ -194,12 +205,13 @@ def test_haketilodb_save_remove(execute_in_page):
'''{
const promise = start_items_transaction(["mappings"], arguments[1])
.then(ctx => save_item(arguments[0], ctx).then(() => ctx))
- .then(finalize_items_transaction)
- .then(get_database_contents);
+ .then(finalize_transaction);
returnval(promise);
}''',
sample_item, sample_files_by_hash)
+ database_contents = get_db_contents(execute_in_page)
+
names = ['README.md', 'report.spdx', 'LICENSES/somelicense.txt', 'hello.js',
'bye.js']
sample_files_list = [sample_files[name] for name in names]
@@ -222,17 +234,18 @@ def test_haketilodb_save_remove(execute_in_page):
# Try removing the items to get an empty database again.
results = [None, None]
for i, item_type in enumerate(['resource', 'mapping']):
- results[i] = execute_in_page(
+ execute_in_page(
f'''{{
const remover = remove_{item_type};
const promise =
start_items_transaction(["{item_type}s"], {{}})
.then(ctx => remover('helloapple', ctx).then(() => ctx))
- .then(finalize_items_transaction)
- .then(get_database_contents);
+ .then(finalize_transaction);
returnval(promise);
}}''')
+ results[i] = get_db_contents(execute_in_page)
+
names = ['README.md', 'report.spdx']
sample_files_list = [sample_files[name] for name in names]
uses_list = [1, 1]
@@ -271,22 +284,48 @@ def test_haketilodb_save_remove(execute_in_page):
},
'files': sample_files_by_hash
}
- database_contents = execute_in_page(
- '''
- initial_data = arguments[0];
- returnval(delete_db().then(() => get_database_contents()));
- ''',
- initial_data)
+
+ clear_indexeddb(execute_in_page)
+ execute_in_page('initial_data = arguments[0];', initial_data)
+ database_contents = get_db_contents(execute_in_page)
+
assert database_contents['resources'] == [sample_resource]
assert database_contents['mappings'] == [sample_mapping]
+@pytest.mark.get_page('https://gotmyowndoma.in')
+def test_haketilodb_settings(driver, execute_in_page):
+ """
+ indexeddb.js facilitates operating on Haketilo's internal database.
+ Verify database assigning/retrieving values of simple "settings" works
+ properly.
+ """
+ execute_in_page(indexeddb_js())
+ mock_broadcast(execute_in_page)
+
+ # Start with no database.
+ clear_indexeddb(execute_in_page)
+
+ assert get_db_contents(execute_in_page)['settings'] == []
+
+ assert execute_in_page('returnval(get_setting("option15"));') == None
+
+ execute_in_page('returnval(set_setting("option15", "disable"));')
+ assert execute_in_page('returnval(get_setting("option15"));') == 'disable'
+
+ execute_in_page('returnval(set_setting("option15", "enable"));')
+ assert execute_in_page('returnval(get_setting("option15"));') == 'enable'
+
test_page_html = '''
<!DOCTYPE html>
<script src="/testpage.js"></script>
+<script>console.log("inline!")</script>
+<script nonce="123456789">console.log("inline nonce!")</script>
<h2>resources</h2>
<ul id="resources"></ul>
<h2>mappings</h2>
<ul id="mappings"></ul>
+<h2>settings</h2>
+<ul id="settings"></ul>
'''
@pytest.mark.ext_data({
@@ -328,15 +367,21 @@ def test_haketilodb_track(driver, execute_in_page, wait_elem_text):
}
for window in reversed(windows):
driver.switch_to.window(window)
- execute_in_page('initial_data = arguments[0];', initial_data)
-
- # See if track_*() functions properly return the already-existing items.
+ try :
+ driver.execute_script('console.log("uuuuuuu");')
+ execute_in_page('initial_data = arguments[0];', initial_data)
+ except:
+ from time import sleep
+ sleep(100000)
+ execute_in_page('returnval(set_setting("option15", "123"));')
+
+ # See if track.*() functions properly return the already-existing items.
execute_in_page(
'''
function update_item(store_name, change)
{
console.log('update', ...arguments);
- const elem_id = `${store_name}_${change.identifier}`;
+ const elem_id = `${store_name}_${change.key}`;
let elem = document.getElementById(elem_id);
elem = elem || document.createElement("li");
elem.id = elem_id;
@@ -348,35 +393,32 @@ def test_haketilodb_track(driver, execute_in_page, wait_elem_text):
let resource_tracking, resource_items, mapping_tracking, mapping_items;
- async function start_tracking()
+ async function start_reporting()
{
- const update_resource = change => update_item("resources", change);
- const update_mapping = change => update_item("mappings", change);
-
- [resource_tracking, resource_items] =
- await track_resources(update_resource);
- [mapping_tracking, mapping_items] =
- await track_mappings(update_mapping);
-
- for (const item of resource_items)
- update_resource({identifier: item.identifier, new_val: item});
- for (const item of mapping_items)
- update_mapping({identifier: item.identifier, new_val: item});
+ for (const store_name of ["resources", "mappings", "settings"]) {
+ [tracking, items] =
+ await track[store_name](ch => update_item(store_name, ch));
+ const prop = store_name === "settings" ? "name" : "identifier";
+ for (const item of items)
+ update_item(store_name, {key: item[prop], new_val: item});
+ }
}
- returnval(start_tracking());
+ returnval(start_reporting());
''')
item_counts = driver.execute_script(
'''
const childcount = id => document.getElementById(id).childElementCount;
- return ["resources", "mappings"].map(childcount);
+ return ["resources", "mappings", "settings"].map(childcount);
''')
- assert item_counts == [1, 1]
+ assert item_counts == [1, 1, 1]
resource_json = driver.find_element_by_id('resources_helloapple').text
mapping_json = driver.find_element_by_id('mappings_helloapple').text
+ setting_json = driver.find_element_by_id('settings_option15').text
assert json.loads(resource_json) == sample_resource
assert json.loads(mapping_json) == sample_mapping
+ assert json.loads(setting_json) == {'name': 'option15', 'value': '123'}
# See if item additions get tracked properly.
driver.switch_to.window(windows[1])
@@ -398,14 +440,17 @@ def test_haketilodb_track(driver, execute_in_page, wait_elem_text):
'files': sample_files_by_hash
}
execute_in_page('returnval(save_items(arguments[0]));', sample_data)
+ execute_in_page('returnval(set_setting("option22", "abc"));')
driver.switch_to.window(windows[0])
driver.implicitly_wait(10)
resource_json = driver.find_element_by_id('resources_helloapple-copy').text
mapping_json = driver.find_element_by_id('mappings_helloapple-copy').text
+ setting_json = driver.find_element_by_id('settings_option22').text
driver.implicitly_wait(0)
assert json.loads(resource_json) == sample_resource2
assert json.loads(mapping_json) == sample_mapping2
+ assert json.loads(setting_json) == {'name': 'option22', 'value': 'abc'}
# See if item deletions get tracked properly.
driver.switch_to.window(windows[1])
@@ -417,7 +462,8 @@ def test_haketilodb_track(driver, execute_in_page, wait_elem_text):
const ctx = await start_items_transaction(store_names, {});
await remove_resource("helloapple", ctx);
await remove_mapping("helloapple-copy", ctx);
- await finalize_items_transaction(ctx);
+ await finalize_transaction(ctx);
+ await set_setting("option22", null);
}
returnval(remove_items());
}''')
@@ -430,7 +476,8 @@ def test_haketilodb_track(driver, execute_in_page, wait_elem_text):
return False
except WebDriverException:
pass
- return True
+ option_text = driver.find_element_by_id('settings_option22').text
+ return json.loads(option_text)['value'] == None
driver.switch_to.window(windows[0])
WebDriverWait(driver, 10).until(condition_items_absent)
diff --git a/test/unit/test_patterns_query_manager.py b/test/unit/test_patterns_query_manager.py
index 8ae7c28..ae1f490 100644
--- a/test/unit/test_patterns_query_manager.py
+++ b/test/unit/test_patterns_query_manager.py
@@ -25,10 +25,9 @@ from selenium.webdriver.support.ui import WebDriverWait
from ..script_loader import load_script
def simple_sample_mapping(patterns, fruit):
- if type(patterns) is list:
- payloads = dict([(p, {'identifier': fruit}) for p in patterns])
- else:
- payloads = {patterns: {'identifier': fruit}}
+ if type(patterns) is not list:
+ patterns = [patterns]
+ payloads = dict([(p, {'identifier': f'{fruit}-{p}'}) for p in patterns])
return {
'source_copyright': [],
'type': 'mapping',
@@ -36,9 +35,13 @@ def simple_sample_mapping(patterns, fruit):
'payloads': payloads
}
-content_script_re = re.compile(r'this.haketilo_pattern_tree = (.*);')
+content_script_tree_re = re.compile(r'this.haketilo_pattern_tree = (.*);')
def extract_tree_data(content_script_text):
- return json.loads(content_script_re.search(content_script_text)[1])
+ return json.loads(content_script_tree_re.search(content_script_text)[1])
+
+content_script_mapping_re = re.compile(r'this.haketilo_mappings = (.*);')
+def extract_mappings_data(content_script_text):
+ return json.loads(content_script_mapping_re.search(content_script_text)[1])
# Fields that are not relevant for testing are omitted from these mapping
# definitions.
@@ -82,7 +85,7 @@ def test_pqm_tree_building(driver, execute_in_page):
return [{}, initial_mappings];
}
- haketilodb.track_mappings = track_mock;
+ haketilodb.track.mappings = track_mock;
let last_script;
let unregister_called = 0;
@@ -104,7 +107,10 @@ def test_pqm_tree_building(driver, execute_in_page):
tree, last_script, unregister_called]);
''',
'https://gotmyowndoma.in/index.html')
- assert found == dict([(m['identifier'], m) for m in sample_mappings[0:2]])
+ best_pattern = 'https://gotmyowndoma.in/index.html'
+ assert found == \
+ dict([(f'inject-{fruit}', {'identifier': f'{fruit}-{best_pattern}'})
+ for fruit in ('banana', 'orange')])
assert tree == extract_tree_data(content_script)
assert deregistrations == 0
@@ -114,12 +120,8 @@ def test_pqm_tree_building(driver, execute_in_page):
execute_in_page(
'''
- for (const mapping of arguments[0]) {
- mappingchange({
- identifier: mapping.identifier,
- new_val: mapping
- });
- }
+ for (const mapping of arguments[0])
+ mappingchange({key: mapping.identifier, new_val: mapping});
''',
sample_mappings[2:])
WebDriverWait(driver, 10).until(condition_mappings_added)
@@ -129,7 +131,8 @@ def test_pqm_tree_building(driver, execute_in_page):
def condition_odd_removed(driver):
last_script = execute_in_page('returnval(last_script);')
- return all([id not in last_script for id in odd])
+ return (all([id not in last_script for id in odd]) and
+ all([id in last_script for id in even]))
def condition_all_removed(driver):
content_script = execute_in_page('returnval(last_script);')
@@ -137,7 +140,7 @@ def test_pqm_tree_building(driver, execute_in_page):
execute_in_page(
'''
- arguments[0].forEach(identifier => mappingchange({identifier}));
+ arguments[0].forEach(identifier => mappingchange({key: identifier}));
''',
odd)
@@ -145,7 +148,7 @@ def test_pqm_tree_building(driver, execute_in_page):
execute_in_page(
'''
- arguments[0].forEach(identifier => mappingchange({identifier}));
+ arguments[0].forEach(identifier => mappingchange({key: identifier}));
''',
even)
@@ -224,7 +227,7 @@ def test_pqm_script_injection(driver, execute_in_page):
const ctx = await start_items_transaction(["mappings"], {});
for (const id of identifiers)
await remove_mapping(id, ctx);
- await finalize_items_transaction(ctx);
+ await finalize_transaction(ctx);
}
returnval(remove_items());
}''',
diff --git a/test/unit/test_policy_deciding.py b/test/unit/test_policy_deciding.py
new file mode 100644
index 0000000..a360537
--- /dev/null
+++ b/test/unit/test_policy_deciding.py
@@ -0,0 +1,121 @@
+# SPDX-License-Identifier: CC0-1.0
+
+"""
+Haketilo unit tests - determining what to do on a given web page
+"""
+
+# This file is part of Haketilo
+#
+# Copyright (C) 2021, Wojtek Kosior <koszko@koszko.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the CC0 1.0 Universal License as published by
+# the Creative Commons Corporation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# CC0 1.0 Universal License for more details.
+
+import re
+from hashlib import sha256
+import pytest
+
+from ..script_loader import load_script
+
+csp_re = re.compile(r'^\S+\s+\S+;(?:\s+\S+\s+\S+;)*$')
+rule_re = re.compile(r'^\s*(?P<src_kind>\S+)\s+(?P<allowed_origins>\S+)$')
+def parse_csp(csp):
+ '''
+ Parsing of CSP string into a dict. A simplified format of CSP is assumed.
+ '''
+ assert csp_re.match(csp)
+
+ result = {}
+
+ for rule in csp.split(';')[:-1]:
+ match = rule_re.match(rule)
+ result[match.group('src_kind')] = match.group('allowed_origins')
+
+ return result
+
+@pytest.mark.get_page('https://gotmyowndoma.in')
+def test_decide_policy(execute_in_page):
+ """
+ policy.js contains code that, using a Pattern Query Tree instance and a URL,
+ decides what Haketilo should do on a page opened at that URL, i.e. whether
+ it should block or allow script execution and whether it should inject its
+ own scripts and which ones. Test that the policy object gets constructed
+ properly.
+ """
+ execute_in_page(load_script('common/policy.js'))
+
+ policy = execute_in_page(
+ '''
+ returnval(decide_policy(pqt.make(), "http://unkno.wn/", true, "abcd"));
+ ''')
+ assert policy['allow'] == True
+ for prop in ('mapping', 'payload', 'nonce', 'csp'):
+ assert prop not in policy
+
+ policy = execute_in_page(
+ '''{
+ const tree = pqt.make();
+ pqt.register(tree, "http://kno.wn", "allowed", {allow: true});
+ returnval(decide_policy(tree, "http://kno.wn/", false, "abcd"));
+ }''')
+ assert policy['allow'] == True
+ assert policy['mapping'] == 'allowed'
+ for prop in ('payload', 'nonce', 'csp'):
+ assert prop not in policy
+
+ policy = execute_in_page(
+ '''
+ returnval(decide_policy(pqt.make(), "http://unkno.wn/", false, "abcd"));
+ '''
+ )
+ assert policy['allow'] == False
+ for prop in ('mapping', 'payload', 'nonce'):
+ assert prop not in policy
+ assert parse_csp(policy['csp']) == {
+ 'prefetch-src': "'none'",
+ 'script-src-attr': "'none'",
+ 'script-src': "'none'",
+ 'script-src-elem': "'none'"
+ }
+
+ policy = execute_in_page(
+ '''{
+ const tree = pqt.make();
+ pqt.register(tree, "http://kno.wn", "disallowed", {allow: false});
+ returnval(decide_policy(tree, "http://kno.wn/", true, "abcd"));
+ }''')
+ assert policy['allow'] == False
+ assert policy['mapping'] == 'disallowed'
+ for prop in ('payload', 'nonce'):
+ assert prop not in policy
+ assert parse_csp(policy['csp']) == {
+ 'prefetch-src': "'none'",
+ 'script-src-attr': "'none'",
+ 'script-src': "'none'",
+ 'script-src-elem': "'none'"
+ }
+
+ policy = execute_in_page(
+ '''{
+ const tree = pqt.make();
+ pqt.register(tree, "http://kno.wn", "m1", {identifier: "res1"});
+ returnval(decide_policy(tree, "http://kno.wn/", true, "abcd"));
+ }''')
+ assert policy['allow'] == False
+ assert policy['mapping'] == 'm1'
+ assert policy['payload'] == {'identifier': 'res1'}
+
+ assert policy['nonce'] == \
+ sha256('m1:res1:http://kno.wn/:abcd'.encode()).digest().hex()
+ assert parse_csp(policy['csp']) == {
+ 'prefetch-src': f"'none'",
+ 'script-src-attr': f"'none'",
+ 'script-src': f"'nonce-{policy['nonce']}'",
+ 'script-src-elem': f"'nonce-{policy['nonce']}'"
+ }
diff --git a/test/unit/test_webrequest.py b/test/unit/test_webrequest.py
new file mode 100644
index 0000000..6af2758
--- /dev/null
+++ b/test/unit/test_webrequest.py
@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: CC0-1.0
+
+"""
+Haketilo unit tests - modifying requests using webRequest API
+"""
+
+# This file is part of Haketilo
+#
+# Copyright (C) 2021, Wojtek Kosior <koszko@koszko.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the CC0 1.0 Universal License as published by
+# the Creative Commons Corporation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# CC0 1.0 Universal License for more details.
+
+import re
+from hashlib import sha256
+import pytest
+
+from ..script_loader import load_script
+
+def webrequest_js():
+ return (load_script('background/webrequest.js',
+ '#IMPORT common/patterns_query_tree.js AS pqt') +
+ ''';
+ // Mock pattern tree.
+ tree = pqt.make();
+ pqt.register(tree, "https://site.with.scripts.block.ed/***",
+ "disallowed", {allow: false});
+ pqt.register(tree, "https://site.with.paylo.ad/***",
+ "somemapping", {identifier: "someresource"});
+
+ // Mock IndexedDB.
+ haketilodb.track.settings =
+ () => [{}, [{name: "default_allow", value: true}]];
+
+ // Mock stream_filter.
+ stream_filter.apply = (details, headers, policy) => headers;
+
+ // Mock secret and start webrequest operations.
+ start("somesecret");
+ ''')
+
+def are_scripts_allowed(driver, nonce=None):
+ return driver.execute_script(
+ '''
+ document.scripts_allowed = false;
+ const script = document.createElement("script");
+ script.innerHTML = "document.scripts_allowed = true;";
+ if (arguments[0])
+ script.setAttribute("nonce", arguments[0]);
+ document.head.append(script);
+ return document.scripts_allowed;
+ ''',
+ nonce)
+
+@pytest.mark.ext_data({'background_script': webrequest_js})
+@pytest.mark.usefixtures('webextension')
+def test_on_headers_received(driver, execute_in_page):
+ for attempt in range(10):
+ driver.get('https://site.with.scripts.block.ed/')
+
+ if not are_scripts_allowed(driver):
+ break
+ assert attempt != 9
+
+ driver.get('https://site.with.scripts.allow.ed/')
+ assert are_scripts_allowed(driver)
+
+ driver.get('https://site.with.paylo.ad/')
+ assert not are_scripts_allowed(driver)
+ source = 'somemapping:someresource:https://site.with.paylo.ad/index.html:somesecret'
+ assert are_scripts_allowed(driver, sha256(source.encode()).digest().hex())
diff --git a/test/world_wide_library.py b/test/world_wide_library.py
index 860c987..43d3512 100644
--- a/test/world_wide_library.py
+++ b/test/world_wide_library.py
@@ -27,13 +27,99 @@ Our helpful little stand-in for the Internet
# file's license. Although I request that you do not make use this code
# in a proprietary program, I am not going to enforce this in court.
+from hashlib import sha256
+from pathlib import Path
+from shutil import rmtree
+from threading import Lock
+
from .misc_constants import here
+served_scripts = {}
+served_scripts_lock = Lock()
+
+def start_serving_script(script_text):
+ """
+ Register given script so that it is served at
+ https://serve.scrip.ts/?sha256=<script's_sha256_sum>
+
+ Returns the URL at which script will be served.
+
+ This function lacks thread safety. Might moght consider fixing this if it
+ turns
+ """
+ sha256sum = sha256(script_text.encode()).digest().hex()
+ served_scripts_lock.acquire()
+ served_scripts[sha256sum] = script_text
+ served_scripts_lock.release()
+
+ return f'https://serve.scrip.ts/?sha256={sha256sum}'
+
+def serve_script(command, get_params, post_params):
+ """
+ info() callback to pass to request-handling code in server.py. Facilitates
+ serving scripts that have been registered with start_serving_script().
+ """
+ served_scripts_lock.acquire()
+ try:
+ script = served_scripts.get(get_params['sha256'][0])
+ finally:
+ served_scripts_lock.release()
+ if script is None:
+ return 404, {}, b''
+
+ return 200, {'Content-Type': 'application/javascript'}, script
+
+def dump_scripts(directory='./injected_scripts'):
+ """
+ Write all scripts that have been registered with start_serving_script()
+ under the provided directory. If the directory already exists, it is wiped
+ beforehand. If it doesn't exist, it is created.
+ """
+ directory = Path(directory)
+ rmtree(directory, ignore_errors=True)
+ directory.mkdir(parents=True)
+
+ served_scripts_lock.acquire()
+ for sha256, script in served_scripts.items():
+ with open(directory / sha256, 'wt') as file:
+ file.write(script)
+ served_scripts_lock.release()
+
catalog = {
- 'http://gotmyowndoma.in': (302, {'location': 'http://gotmyowndoma.in/index.html'}, None),
- 'http://gotmyowndoma.in/': (302, {'location': 'http://gotmyowndoma.in/index.html'}, None),
- 'http://gotmyowndoma.in/index.html': (200, {}, here / 'data' / 'pages' / 'gotmyowndomain.html'),
- 'https://gotmyowndoma.in': (302, {'location': 'https://gotmyowndoma.in/index.html'}, None),
- 'https://gotmyowndoma.in/': (302, {'location': 'https://gotmyowndoma.in/index.html'}, None),
- 'https://gotmyowndoma.in/index.html': (200, {}, here / 'data' / 'pages' / 'gotmyowndomain_https.html')
+ 'http://gotmyowndoma.in':
+ (302, {'location': 'http://gotmyowndoma.in/index.html'}, None),
+ 'http://gotmyowndoma.in/':
+ (302, {'location': 'http://gotmyowndoma.in/index.html'}, None),
+ 'http://gotmyowndoma.in/index.html':
+ (200, {}, here / 'data' / 'pages' / 'gotmyowndomain.html'),
+
+ 'https://gotmyowndoma.in':
+ (302, {'location': 'https://gotmyowndoma.in/index.html'}, None),
+ 'https://gotmyowndoma.in/':
+ (302, {'location': 'https://gotmyowndoma.in/index.html'}, None),
+ 'https://gotmyowndoma.in/index.html':
+ (200, {}, here / 'data' / 'pages' / 'gotmyowndomain_https.html'),
+
+ 'https://serve.scrip.ts/': serve_script,
+
+ 'https://site.with.scripts.block.ed':
+ (302, {'location': 'https://site.with.scripts.block.ed/index.html'}, None),
+ 'https://site.with.scripts.block.ed/':
+ (302, {'location': 'https://site.with.scripts.block.ed/index.html'}, None),
+ 'https://site.with.scripts.block.ed/index.html':
+ (200, {}, here / 'data' / 'pages' / 'gotmyowndomain_https.html'),
+
+ 'https://site.with.scripts.allow.ed':
+ (302, {'location': 'https://site.with.scripts.allow.ed/index.html'}, None),
+ 'https://site.with.scripts.allow.ed/':
+ (302, {'location': 'https://site.with.scripts.allow.ed/index.html'}, None),
+ 'https://site.with.scripts.allow.ed/index.html':
+ (200, {}, here / 'data' / 'pages' / 'gotmyowndomain_https.html'),
+
+ 'https://site.with.paylo.ad':
+ (302, {'location': 'https://site.with.paylo.ad/index.html'}, None),
+ 'https://site.with.paylo.ad/':
+ (302, {'location': 'https://site.with.paylo.ad/index.html'}, None),
+ 'https://site.with.paylo.ad/index.html':
+ (200, {}, here / 'data' / 'pages' / 'gotmyowndomain_https.html')
}
generated by cgit