From fd9f2fc4783cc606734e61116185c032a63d54a0 Mon Sep 17 00:00:00 2001 From: Wojtek Kosior Date: Wed, 16 Feb 2022 22:01:38 +0100 Subject: fix out-of-source builds --- Makefile.in | 50 +-- build.sh | 2 - pytest.ini | 20 - pytest.ini.in | 22 + test/__init__.py | 2 - test/__main__.py | 83 ---- test/conftest.py | 187 -------- test/data/pages/gotmyowndomain.html | 35 -- test/data/pages/gotmyowndomain_https.html | 35 -- test/data/pages/scripts_to_block_1.html | 45 -- test/default_profiles/icecat_empty/extensions.json | 1 - test/extension_crafting.py | 215 --------- test/haketilo_test/__init__.py | 2 + test/haketilo_test/__main__.py | 83 ++++ test/haketilo_test/conftest.py | 187 ++++++++ test/haketilo_test/data/pages/gotmyowndomain.html | 35 ++ .../data/pages/gotmyowndomain_https.html | 35 ++ .../data/pages/scripts_to_block_1.html | 45 ++ .../default_profiles/icecat_empty/extensions.json | 1 + test/haketilo_test/extension_crafting.py | 215 +++++++++ test/haketilo_test/misc_constants.py | 82 ++++ test/haketilo_test/profiles.py | 116 +++++ test/haketilo_test/proxy_core.py | 141 ++++++ test/haketilo_test/script_loader.py | 72 +++ test/haketilo_test/server.py | 112 +++++ test/haketilo_test/test_integration.py | 47 ++ test/haketilo_test/unit/__init__.py | 2 + test/haketilo_test/unit/test_CORS_bypass_server.py | 109 +++++ test/haketilo_test/unit/test_basic.py | 77 ++++ test/haketilo_test/unit/test_broadcast.py | 175 ++++++++ test/haketilo_test/unit/test_content.py | 190 ++++++++ .../unit/test_default_policy_dialog.py | 49 +++ test/haketilo_test/unit/test_dialog.py | 143 ++++++ test/haketilo_test/unit/test_indexeddb.py | 490 +++++++++++++++++++++ .../unit/test_indexeddb_files_server.py | 171 +++++++ test/haketilo_test/unit/test_install.py | 423 ++++++++++++++++++ test/haketilo_test/unit/test_item_list.py | 280 ++++++++++++ test/haketilo_test/unit/test_item_preview.py | 208 +++++++++ test/haketilo_test/unit/test_patterns.py | 152 +++++++ .../unit/test_patterns_query_manager.py | 307 +++++++++++++ .../haketilo_test/unit/test_patterns_query_tree.py | 474 ++++++++++++++++++++ test/haketilo_test/unit/test_payload_create.py | 248 +++++++++++ test/haketilo_test/unit/test_policy_deciding.py | 135 ++++++ test/haketilo_test/unit/test_policy_enforcing.py | 114 +++++ test/haketilo_test/unit/test_popup.py | 257 +++++++++++ test/haketilo_test/unit/test_repo_query.py | 274 ++++++++++++ test/haketilo_test/unit/test_repo_query_cacher.py | 130 ++++++ test/haketilo_test/unit/test_settings.py | 63 +++ test/haketilo_test/unit/test_text_entry_list.py | 387 ++++++++++++++++ test/haketilo_test/unit/test_webrequest.py | 68 +++ test/haketilo_test/unit/utils.py | 293 ++++++++++++ test/haketilo_test/world_wide_library.py | 270 ++++++++++++ test/misc_constants.py | 82 ---- test/profiles.py | 116 ----- test/proxy_core.py | 141 ------ test/script_loader.py | 72 --- test/server.py | 111 ----- test/test_integration.py | 47 -- test/unit/__init__.py | 2 - test/unit/test_CORS_bypass_server.py | 109 ----- test/unit/test_basic.py | 77 ---- test/unit/test_broadcast.py | 175 -------- test/unit/test_content.py | 190 -------- test/unit/test_default_policy_dialog.py | 49 --- test/unit/test_dialog.py | 143 ------ test/unit/test_indexeddb.py | 490 --------------------- test/unit/test_indexeddb_files_server.py | 171 ------- test/unit/test_install.py | 423 ------------------ test/unit/test_item_list.py | 280 ------------ test/unit/test_item_preview.py | 208 --------- test/unit/test_patterns.py | 152 ------- test/unit/test_patterns_query_manager.py | 307 ------------- test/unit/test_patterns_query_tree.py | 474 -------------------- test/unit/test_payload_create.py | 248 ----------- test/unit/test_policy_deciding.py | 135 ------ test/unit/test_policy_enforcing.py | 114 ----- test/unit/test_popup.py | 257 ----------- test/unit/test_repo_query.py | 274 ------------ test/unit/test_repo_query_cacher.py | 130 ------ test/unit/test_settings.py | 63 --- test/unit/test_text_entry_list.py | 387 ---------------- test/unit/test_webrequest.py | 68 --- test/unit/utils.py | 293 ------------ test/world_wide_library.py | 270 ------------ 84 files changed, 6706 insertions(+), 6711 deletions(-) delete mode 100644 pytest.ini create mode 100644 pytest.ini.in delete mode 100644 test/__init__.py delete mode 100644 test/__main__.py delete mode 100644 test/conftest.py delete mode 100644 test/data/pages/gotmyowndomain.html delete mode 100644 test/data/pages/gotmyowndomain_https.html delete mode 100644 test/data/pages/scripts_to_block_1.html delete mode 100644 test/default_profiles/icecat_empty/extensions.json delete mode 100644 test/extension_crafting.py create mode 100644 test/haketilo_test/__init__.py create mode 100644 test/haketilo_test/__main__.py create mode 100644 test/haketilo_test/conftest.py create mode 100644 test/haketilo_test/data/pages/gotmyowndomain.html create mode 100644 test/haketilo_test/data/pages/gotmyowndomain_https.html create mode 100644 test/haketilo_test/data/pages/scripts_to_block_1.html create mode 100644 test/haketilo_test/default_profiles/icecat_empty/extensions.json create mode 100644 test/haketilo_test/extension_crafting.py create mode 100644 test/haketilo_test/misc_constants.py create mode 100755 test/haketilo_test/profiles.py create mode 100644 test/haketilo_test/proxy_core.py create mode 100644 test/haketilo_test/script_loader.py create mode 100755 test/haketilo_test/server.py create mode 100644 test/haketilo_test/test_integration.py create mode 100644 test/haketilo_test/unit/__init__.py create mode 100644 test/haketilo_test/unit/test_CORS_bypass_server.py create mode 100644 test/haketilo_test/unit/test_basic.py create mode 100644 test/haketilo_test/unit/test_broadcast.py create mode 100644 test/haketilo_test/unit/test_content.py create mode 100644 test/haketilo_test/unit/test_default_policy_dialog.py create mode 100644 test/haketilo_test/unit/test_dialog.py create mode 100644 test/haketilo_test/unit/test_indexeddb.py create mode 100644 test/haketilo_test/unit/test_indexeddb_files_server.py create mode 100644 test/haketilo_test/unit/test_install.py create mode 100644 test/haketilo_test/unit/test_item_list.py create mode 100644 test/haketilo_test/unit/test_item_preview.py create mode 100644 test/haketilo_test/unit/test_patterns.py create mode 100644 test/haketilo_test/unit/test_patterns_query_manager.py create mode 100644 test/haketilo_test/unit/test_patterns_query_tree.py create mode 100644 test/haketilo_test/unit/test_payload_create.py create mode 100644 test/haketilo_test/unit/test_policy_deciding.py create mode 100644 test/haketilo_test/unit/test_policy_enforcing.py create mode 100644 test/haketilo_test/unit/test_popup.py create mode 100644 test/haketilo_test/unit/test_repo_query.py create mode 100644 test/haketilo_test/unit/test_repo_query_cacher.py create mode 100644 test/haketilo_test/unit/test_settings.py create mode 100644 test/haketilo_test/unit/test_text_entry_list.py create mode 100644 test/haketilo_test/unit/test_webrequest.py create mode 100644 test/haketilo_test/unit/utils.py create mode 100644 test/haketilo_test/world_wide_library.py delete mode 100644 test/misc_constants.py delete mode 100755 test/profiles.py delete mode 100644 test/proxy_core.py delete mode 100644 test/script_loader.py delete mode 100755 test/server.py delete mode 100644 test/test_integration.py delete mode 100644 test/unit/__init__.py delete mode 100644 test/unit/test_CORS_bypass_server.py delete mode 100644 test/unit/test_basic.py delete mode 100644 test/unit/test_broadcast.py delete mode 100644 test/unit/test_content.py delete mode 100644 test/unit/test_default_policy_dialog.py delete mode 100644 test/unit/test_dialog.py delete mode 100644 test/unit/test_indexeddb.py delete mode 100644 test/unit/test_indexeddb_files_server.py delete mode 100644 test/unit/test_install.py delete mode 100644 test/unit/test_item_list.py delete mode 100644 test/unit/test_item_preview.py delete mode 100644 test/unit/test_patterns.py delete mode 100644 test/unit/test_patterns_query_manager.py delete mode 100644 test/unit/test_patterns_query_tree.py delete mode 100644 test/unit/test_payload_create.py delete mode 100644 test/unit/test_policy_deciding.py delete mode 100644 test/unit/test_policy_enforcing.py delete mode 100644 test/unit/test_popup.py delete mode 100644 test/unit/test_repo_query.py delete mode 100644 test/unit/test_repo_query_cacher.py delete mode 100644 test/unit/test_settings.py delete mode 100644 test/unit/test_text_entry_list.py delete mode 100644 test/unit/test_webrequest.py delete mode 100644 test/unit/utils.py delete mode 100644 test/world_wide_library.py diff --git a/Makefile.in b/Makefile.in index 28ccb85..862c610 100644 --- a/Makefile.in +++ b/Makefile.in @@ -17,9 +17,10 @@ VPATH = <> version = <> extension_files = background/ common/ content/ html/ licenses/ \ - copyright default_settings.json manifest.json + icons/ copyright default_settings.json manifest.json -metafiles = build.sh configure Makefile.in process_html_file.sh README.txt \ + +metafiles = build.sh configure Makefile.in compute_scripts.awk README.txt \ re-generate_icons.sh shell_utils.sh upload_amo.sh write_makefile.sh # Configuration gets included here by write_makefile.sh @@ -47,11 +48,11 @@ chromium: chromium-build.zip unpacked: $(default_target)-unpacked Makefile all-unpacked: mozilla-unpacked chromium-unpacked %-unpacked: $(extension_files) icons/haketilo16.png - $(srcdir)/build.sh $* $(srcdir) $(UPDATE_URL) + "$(srcdir)/build.sh" $* $(srcdir) $(UPDATE_URL) install install-strip: $(default_target)-unpacked cp -R $(default_target)-unpacked \ - "$(DESTDIR)/{6fe13369-88e9-440f-b837-5012fb3bedec}" + "$(DESTDIR)/{6fe13369-88e9-440f-b837-5012fb3bedec}" uninstall: rm -r "$(DESTDIR)/{6fe13369-88e9-440f-b837-5012fb3bedec}" @@ -59,37 +60,37 @@ uninstall: %-build.zip: %-unpacked Makefile cd $< && zip -q -r ../$@ * -test/: - mkdir $@ - -test/certs/: | test/ +certs/: mkdir $@ -test/certs/%.key: | test/certs/ +certs/%.key: | certs/ openssl genrsa -out $@ 2048 -test/certs/rootCA.pem: test/certs/rootCA.key +certs/rootCA.pem: certs/rootCA.key openssl req -x509 -new -nodes -key $< -days 1024 -out $@ \ -subj "/CN=Haketilo Test" -test: test/certs/rootCA.pem test/certs/site.key $(default_target)-build.zip - MOZ_HEADLESS=whatever $(PYTHON) -m pytest +pytest.ini: pytest.ini.in + sed "s|<>|$(srcdir)|" <$< > $@ -test-environment: test/certs/rootCA.pem test/certs/site.key - python3 -m test +test: certs/rootCA.pem certs/site.key $(default_target)-build.zip \ + pytest.ini + PYTHONPYCACHEPREFIX=$$(pwd)/test__pycache__ MOZ_HEADLESS=whatever \ + "$(PYTHON)" -m pytest -test-environment-with-haketilo: test/certs/rootCA.pem test/certs/site.key \ +test-environment: certs/rootCA.pem certs/site.key + "$(PYTHON)" -m test + +test-environment-with-haketilo: certs/rootCA.pem certs/site.key \ $(default_target)-build.zip - python3 -m test --load-haketilo + "$(PYTHON)" -m test --load-haketilo # helper targets clean mostlyclean: rm -rf mozilla-unpacked chromium-unpacked haketilo-[1-9]* rm -f mozilla-build.zip chromium-build.zip exports_init.js - rm -rf test/certs - rm -rf $$(find . -name geckodriver.log) - rm -rf $$(find . -type d -name __pycache__) - rm -rf $$(find . -type d -name injected_scripts) + rm -rf pytest.ini certs injected_scripts geckodriver.log + rm -rf certs/ test__pycache__/ .pytest_cache/ distclean: clean rm -f Makefile config.status record.conf @@ -99,14 +100,6 @@ maintainer-clean: distclean @echo 'deletes files that may need special tools to rebuild.' rm -f "$(srcdir)"/icons/*.png -dist: $(extension_files) $(metafiles) icons/haketilo16.png - test -d haketilo-$(version) || mkdir haketilo-$(version) - for file in $(extension_files) $(metafiles) icons/; do \ - cp -R "$(srcdir)"/$$file haketilo-$(version); \ - done - tar cf haketilo-$(version).tar haketilo-$(version) - gzip haketilo-$(version).tar - # Files for constructing the makefile Makefile: config.status Makefile.in record.conf ./config.status @@ -115,6 +108,7 @@ config.status: write_makefile.sh cp "$(srcdir)"/write_makefile.sh config.status # Unused GNU-specified targets +dist: install-html: install-dvi: install-pdf: diff --git a/build.sh b/build.sh index 1486073..c38fc60 100755 --- a/build.sh +++ b/build.sh @@ -15,8 +15,6 @@ set -e -. ./shell_utils.sh - print_usage() { printf 'usage: %s mozilla|chromium [source directory] [update url]\n' \ "$0" >&2 diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 1057a9a..0000000 --- a/pytest.ini +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env pytest - -# This file is part of Haketilo -# -# Copyright (C) 2021, Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -[pytest] -markers = - ext_data: define a custom testing extension for `webextension` fixture. - get_page: define a url the `driver` fixture should navigate the browser to. - second_driver: tell `driver` fixture to spawn a separate browser instance fr this test. diff --git a/pytest.ini.in b/pytest.ini.in new file mode 100644 index 0000000..ea0b58f --- /dev/null +++ b/pytest.ini.in @@ -0,0 +1,22 @@ +#!/usr/bin/env pytest + +# This file is part of Haketilo +# +# Copyright (C) 2021, 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +[pytest] +markers = + ext_data: define a custom testing extension for `webextension` fixture. + get_page: define a url the `driver` fixture should navigate the browser to. + second_driver: tell `driver` fixture to spawn a separate browser instance for this test. +testpaths = + <>/test/haketilo_test diff --git a/test/__init__.py b/test/__init__.py deleted file mode 100644 index 2b351bb..0000000 --- a/test/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 -# Copyright (C) 2021 Wojtek Kosior diff --git a/test/__main__.py b/test/__main__.py deleted file mode 100644 index 7afda55..0000000 --- a/test/__main__.py +++ /dev/null @@ -1,83 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-or-later - -""" -Run a Firefox-type browser with WebDriver attached and Python console open -""" - -# This file is part of Haketilo. -# -# Copyright (C) 2021 jahoti -# Copyright (C) 2021 Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# -# I, Wojtek Kosior, thereby promise not to sue for violation of this -# file's license. Although I request that you do not make use of this code -# in a proprietary program, I am not going to enforce this in court. - -import sys -import time -import code -from rlcompleter import Completer -import readline - -from .server import do_an_internet -from .misc_constants import * -from .profiles import firefox_safe_mode -from .extension_crafting import get_extension_base_url - -def fail(msg, error_code): - print('Error:', msg) - print('Usage:', sys.argv[0], '[--load-haketilo]', '[certificates_directory] [proxy_port]') - sys.exit(error_code) - -load_haketilo = False -argv_idx = 1 -if len(sys.argv) > argv_idx and sys.argv[argv_idx] == '--load-haketilo': - load_haketilo = True - argv_idx += 1 - -certdir = Path(sys.argv[argv_idx]).resolve() if len(sys.argv) > argv_idx \ - else default_cert_dir - -if not certdir.is_dir(): - fail('selected certificate directory does not exist.', 2) - -argv_idx += 1 - -port = sys.argv[argv_idx] if len(sys.argv) > argv_idx \ - else str(default_proxy_port) - -if not port.isnumeric(): - fail('port must be an integer.', 3) - -httpd = do_an_internet(certdir, int(port)) -driver = firefox_safe_mode(proxy_port=int(port)) - -if load_haketilo: - driver.install_addon(str(here.parent / 'mozilla-build.zip'), temporary=True) - driver.get(get_extension_base_url(driver) + 'html/settings.html') - -print("You can now control the browser through 'driver' object") - -# Here we enable readline-enhanced editing: -# https://stackoverflow.com/questions/35115208/is-there-any-way-to-combine-readline-rlcompleter-and-interactiveconsole-in-pytho#answer-35116399 -readline.parse_and_bind('tab: complete'); -console_locals = globals() -readline.set_completer(Completer(console_locals).complete) -code.InteractiveConsole(locals=globals()).interact() - -driver.quit() -httpd.shutdown() diff --git a/test/conftest.py b/test/conftest.py deleted file mode 100644 index c0ece61..0000000 --- a/test/conftest.py +++ /dev/null @@ -1,187 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -""" -Common fixtures for Haketilo unit tests -""" - -# This file is part of Haketilo. -# -# Copyright (C) 2021 Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# I, Wojtek Kosior, thereby promise not to sue for violation of this file's -# license. Although I request that you do not make use of this code in a -# proprietary program, I am not going to enforce this in court. - -import pytest -from pathlib import Path -from tempfile import TemporaryDirectory -from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC - -from .profiles import firefox_safe_mode -from .server import do_an_internet -from .extension_crafting import make_extension -from .world_wide_library import start_serving_script, dump_scripts -from .misc_constants import here - -@pytest.fixture(scope="session") -def proxy(): - httpd = do_an_internet() - yield httpd - httpd.shutdown() - -@pytest.fixture(scope="session") -def _driver(proxy): - with firefox_safe_mode() as driver: - yield driver - driver.quit() - -def close_all_but_one_window(driver): - while len(driver.window_handles) > 1: - driver.switch_to.window(driver.window_handles[-1]) - driver.close() - driver.switch_to.window(driver.window_handles[0]) - -@pytest.fixture() -def driver(_driver, request): - nav_target = request.node.get_closest_marker('get_page') - nav_target = nav_target.args[0] if nav_target else 'about:blank' - - second_driver = request.node.get_closest_marker('second_driver') - - if second_driver: - with firefox_safe_mode() as _driver: - _driver.get(nav_target) - yield _driver - _driver.quit() - else: - close_all_but_one_window(_driver) - _driver.get(nav_target) - _driver.implicitly_wait(0) - yield _driver - -@pytest.fixture() -def webextension(driver, request): - ext_data = request.node.get_closest_marker('ext_data') - if ext_data is None: - raise Exception('"webextension" fixture requires "ext_data" marker to be set') - ext_data = ext_data.args[0].copy() - - navigate_to = ext_data.get('navigate_to') - if navigate_to is not None: - del ext_data['navigate_to'] - - driver.get('https://gotmyowndoma.in/') - ext_path = make_extension(Path(driver.firefox_profile.path), **ext_data) - addon_id = driver.install_addon(str(ext_path), temporary=True) - get_url = lambda d: d.execute_script('return window.ext_page_url;') - ext_page_url = WebDriverWait(driver, 10).until(get_url) - driver.get(ext_page_url) - - if navigate_to is not None: - driver.get(driver.current_url.replace('testpage.html', navigate_to)) - - yield - - # Unloading an extension might cause its windows to vanish. Make sure - # there's at least one window navigated to some other page before - # uninstalling the addon. Otherwise, we could be left with a windowless - # browser :c - driver.switch_to.window(driver.window_handles[-1]) - driver.get('https://gotmyowndoma.in/') - driver.uninstall_addon(addon_id) - ext_path.unlink() - -@pytest.fixture() -def haketilo(driver): - addon_id = driver.install_addon(str(here.parent / 'mozilla-build.zip'), - temporary=True) - - yield - - driver.uninstall_addon(addon_id) - -script_injector_script = '''\ -/* - * Selenium by default executes scripts in some weird one-time context. We want - * separately-loaded scripts to be able to access global variables defined - * before, including those declared with `const` or `let`. To achieve that, we - * run our scripts by injecting them into the page with a - - diff --git a/test/data/pages/gotmyowndomain_https.html b/test/data/pages/gotmyowndomain_https.html deleted file mode 100644 index f602950..0000000 --- a/test/data/pages/gotmyowndomain_https.html +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - Schrodinger's Document - - - A nice, simple page for testing (using HTTPS). - - - diff --git a/test/data/pages/scripts_to_block_1.html b/test/data/pages/scripts_to_block_1.html deleted file mode 100644 index 164979d..0000000 --- a/test/data/pages/scripts_to_block_1.html +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - - - - - Click Meee! - - - - - - diff --git a/test/default_profiles/icecat_empty/extensions.json b/test/default_profiles/icecat_empty/extensions.json deleted file mode 100644 index 5f74ff3..0000000 --- a/test/default_profiles/icecat_empty/extensions.json +++ /dev/null @@ -1 +0,0 @@ -{"schemaVersion":25,"addons":[{"id":"jid1-KtlZuoiikVfFew@jetpack","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/jid1-KtlZuoiikVfFew@jetpack"},{"id":"uBlock0@raymondhill.net","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/uBlock0@raymondhill.net.xpi"},{"id":"SubmitMe@0xbeef.coffee","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/SubmitMe@0xbeef.coffee"},{"id":"FreeUSPS@0xbeef.coffee","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/FreeUSPS@0xbeef.coffee"},{"id":"tortm-browser-button@jeremybenthum","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/tortm-browser-button@jeremybenthum"},{"id":"tprb.addon@searxes.danwin1210.me","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/tprb.addon@searxes.danwin1210.me"},{"id":"SimpleSumOfUs@0xbeef.coffee","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/SimpleSumOfUs@0xbeef.coffee"}]} \ No newline at end of file diff --git a/test/extension_crafting.py b/test/extension_crafting.py deleted file mode 100644 index 680c45e..0000000 --- a/test/extension_crafting.py +++ /dev/null @@ -1,215 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -""" -Making temporary WebExtensions for use in the test suite -""" - -# This file is part of Haketilo. -# -# Copyright (C) 2021, 2022 Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# I, Wojtek Kosior, thereby promise not to sue for violation of this file's -# license. Although I request that you do not make use of this code in a -# proprietary program, I am not going to enforce this in court. - -import json -import zipfile -import re -import shutil -import subprocess - -from pathlib import Path -from uuid import uuid4 -from tempfile import TemporaryDirectory - -from selenium.webdriver.support.ui import WebDriverWait -from selenium.common.exceptions import NoSuchElementException - -from .misc_constants import * - -class ManifestTemplateValueToFill: - pass - -def manifest_template(): - return { - 'manifest_version': 2, - 'name': 'Haketilo test extension', - 'version': '1.0', - 'applications': { - 'gecko': { - 'id': ManifestTemplateValueToFill(), - 'strict_min_version': '60.0' - } - }, - 'permissions': [ - 'contextMenus', - 'webRequest', - 'webRequestBlocking', - 'activeTab', - 'notifications', - 'sessions', - 'storage', - 'tabs', - '', - 'unlimitedStorage' - ], - 'content_security_policy': "object-src 'none'; script-src 'self' https://serve.scrip.ts;", - 'web_accessible_resources': ['testpage.html'], - 'options_ui': { - 'page': 'testpage.html', - 'open_in_tab': True - }, - 'background': { - 'persistent': True, - 'scripts': ['__open_test_page.js', 'background.js'] - }, - 'content_scripts': [ - { - 'run_at': 'document_start', - 'matches': [''], - 'match_about_blank': True, - 'all_frames': True, - 'js': ['content.js'] - } - ] - } - -class ExtraHTML: - def __init__(self, html_path, append={}, wrap_into_htmldoc=True): - self.html_path = html_path - self.append = append - self.wrap_into_htmldoc = wrap_into_htmldoc - - def add_to_xpi(self, xpi, tmpdir=None): - if tmpdir is None: - with TemporaryDirectory() as tmpdir: - return self.add_to_xpi(xpi, tmpdir) - - append_flags = [] - for filename, code in self.append.items(): - append_flags.extend(['-A', f'{filename}:{code}']) - - awk = subprocess.run( - ['awk', '-f', awk_script_name, '--', *unit_test_defines, - *append_flags, '-H', self.html_path, '--write-js-deps', - '--output=files-to-copy', f'--output-dir={tmpdir}'], - stdout=subprocess.PIPE, cwd=script_root, check=True - ) - - for path in filter(None, awk.stdout.decode().split('\n')): - xpi.write(script_root / path, path) - - tmpdir = Path(tmpdir) - for path in tmpdir.rglob('*'): - relpath = str(path.relative_to(tmpdir)) - if not path.is_dir() and relpath != self.html_path: - xpi.write(path, relpath) - - with open(tmpdir / self.html_path, 'rt') as html_file: - html = html_file.read() - if self.wrap_into_htmldoc: - html = f'{html}' - xpi.writestr(self.html_path, html) - -default_background_script = '' -default_content_script = '' -default_test_page = ''' - - - - Extension's options page for testing - - -

Extension's options page for testing

- - -''' - -open_test_page_script = '''(() => { -const page_url = browser.runtime.getURL("testpage.html"); -const execute_details = { - code: `window.wrappedJSObject.ext_page_url=${JSON.stringify(page_url)};` -}; -browser.tabs.query({currentWindow: true, active: true}) - .then(t => browser.tabs.executeScript(t.id, execute_details)); -})();''' - -def make_extension(destination_dir, - background_script=default_background_script, - content_script=default_content_script, - test_page=default_test_page, - extra_files={}, extra_html=[]): - if not hasattr(extra_html, '__iter__'): - extra_html = [extra_html] - manifest = manifest_template() - extension_id = '{%s}' % uuid4() - manifest['applications']['gecko']['id'] = extension_id - files = { - 'manifest.json' : json.dumps(manifest), - '__open_test_page.js': open_test_page_script, - 'background.js' : background_script, - 'content.js' : content_script, - 'testpage.html' : test_page, - **extra_files - } - destination_path = destination_dir / f'{extension_id}.xpi' - with zipfile.ZipFile(destination_path, 'x') as xpi: - for filename, contents in files.items(): - if hasattr(contents, '__call__'): - contents = contents() - xpi.writestr(filename, contents) - for html in extra_html: - html.add_to_xpi(xpi) - - return destination_path - -extract_base_url_re = re.compile(r'^(.*)manifest.json$') - -def get_extension_base_url(driver): - """ - Extension's internall UUID is not directly exposed in Selenium. Instead, we - can navigate to about:debugging and inspect the manifest URL present there - to get the base url like: - moz-extension://b225c78f-d108-4caa-8406-f38b37d8dee5/ - which can then be used to navigate to extension-bundled pages. - """ - # For newer Firefoxes - driver.get('about:debugging#/runtime/this-firefox') - - def get_manifest_link_newer_ff(driver): - try: - return driver.find_element_by_class_name('qa-manifest-url') - except NoSuchElementException: - pass - - try: - details = driver.find_element_by_class_name('error-page-details') - except NoSuchElementException: - return False - - if '#/runtime/this-firefox' in details.text: - return "not_newer_ff" - - manifest_link = WebDriverWait(driver, 10).until(get_manifest_link_newer_ff) - - if manifest_link == "not_newer_ff": - driver.get("about:debugging#addons") - driver.implicitly_wait(10) - manifest_link = driver.find_element_by_class_name('manifest-url') - driver.implicitly_wait(0) - - manifest_url = manifest_link.get_attribute('href') - return extract_base_url_re.match(manifest_url).group(1) diff --git a/test/haketilo_test/__init__.py b/test/haketilo_test/__init__.py new file mode 100644 index 0000000..2b351bb --- /dev/null +++ b/test/haketilo_test/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: CC0-1.0 +# Copyright (C) 2021 Wojtek Kosior diff --git a/test/haketilo_test/__main__.py b/test/haketilo_test/__main__.py new file mode 100644 index 0000000..7afda55 --- /dev/null +++ b/test/haketilo_test/__main__.py @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +""" +Run a Firefox-type browser with WebDriver attached and Python console open +""" + +# This file is part of Haketilo. +# +# Copyright (C) 2021 jahoti +# Copyright (C) 2021 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this +# file's license. Although I request that you do not make use of this code +# in a proprietary program, I am not going to enforce this in court. + +import sys +import time +import code +from rlcompleter import Completer +import readline + +from .server import do_an_internet +from .misc_constants import * +from .profiles import firefox_safe_mode +from .extension_crafting import get_extension_base_url + +def fail(msg, error_code): + print('Error:', msg) + print('Usage:', sys.argv[0], '[--load-haketilo]', '[certificates_directory] [proxy_port]') + sys.exit(error_code) + +load_haketilo = False +argv_idx = 1 +if len(sys.argv) > argv_idx and sys.argv[argv_idx] == '--load-haketilo': + load_haketilo = True + argv_idx += 1 + +certdir = Path(sys.argv[argv_idx]).resolve() if len(sys.argv) > argv_idx \ + else default_cert_dir + +if not certdir.is_dir(): + fail('selected certificate directory does not exist.', 2) + +argv_idx += 1 + +port = sys.argv[argv_idx] if len(sys.argv) > argv_idx \ + else str(default_proxy_port) + +if not port.isnumeric(): + fail('port must be an integer.', 3) + +httpd = do_an_internet(certdir, int(port)) +driver = firefox_safe_mode(proxy_port=int(port)) + +if load_haketilo: + driver.install_addon(str(here.parent / 'mozilla-build.zip'), temporary=True) + driver.get(get_extension_base_url(driver) + 'html/settings.html') + +print("You can now control the browser through 'driver' object") + +# Here we enable readline-enhanced editing: +# https://stackoverflow.com/questions/35115208/is-there-any-way-to-combine-readline-rlcompleter-and-interactiveconsole-in-pytho#answer-35116399 +readline.parse_and_bind('tab: complete'); +console_locals = globals() +readline.set_completer(Completer(console_locals).complete) +code.InteractiveConsole(locals=globals()).interact() + +driver.quit() +httpd.shutdown() diff --git a/test/haketilo_test/conftest.py b/test/haketilo_test/conftest.py new file mode 100644 index 0000000..9103ac8 --- /dev/null +++ b/test/haketilo_test/conftest.py @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +""" +Common fixtures for Haketilo unit tests +""" + +# This file is part of Haketilo. +# +# Copyright (C) 2021 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this file's +# license. Although I request that you do not make use of this code in a +# proprietary program, I am not going to enforce this in court. + +import pytest +from pathlib import Path +from tempfile import TemporaryDirectory +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC + +from .profiles import firefox_safe_mode +from .server import do_an_internet +from .extension_crafting import make_extension +from .world_wide_library import start_serving_script, dump_scripts +from .misc_constants import proj_root + +@pytest.fixture(scope="session") +def proxy(): + httpd = do_an_internet() + yield httpd + httpd.shutdown() + +@pytest.fixture(scope="session") +def _driver(proxy): + with firefox_safe_mode() as driver: + yield driver + driver.quit() + +def close_all_but_one_window(driver): + while len(driver.window_handles) > 1: + driver.switch_to.window(driver.window_handles[-1]) + driver.close() + driver.switch_to.window(driver.window_handles[0]) + +@pytest.fixture() +def driver(_driver, request): + nav_target = request.node.get_closest_marker('get_page') + nav_target = nav_target.args[0] if nav_target else 'about:blank' + + second_driver = request.node.get_closest_marker('second_driver') + + if second_driver: + with firefox_safe_mode() as _driver: + _driver.get(nav_target) + yield _driver + _driver.quit() + else: + close_all_but_one_window(_driver) + _driver.get(nav_target) + _driver.implicitly_wait(0) + yield _driver + +@pytest.fixture() +def webextension(driver, request): + ext_data = request.node.get_closest_marker('ext_data') + if ext_data is None: + raise Exception('"webextension" fixture requires "ext_data" marker to be set') + ext_data = ext_data.args[0].copy() + + navigate_to = ext_data.get('navigate_to') + if navigate_to is not None: + del ext_data['navigate_to'] + + driver.get('https://gotmyowndoma.in/') + ext_path = make_extension(Path(driver.firefox_profile.path), **ext_data) + addon_id = driver.install_addon(str(ext_path), temporary=True) + get_url = lambda d: d.execute_script('return window.ext_page_url;') + ext_page_url = WebDriverWait(driver, 10).until(get_url) + driver.get(ext_page_url) + + if navigate_to is not None: + driver.get(driver.current_url.replace('testpage.html', navigate_to)) + + yield + + # Unloading an extension might cause its windows to vanish. Make sure + # there's at least one window navigated to some other page before + # uninstalling the addon. Otherwise, we could be left with a windowless + # browser :c + driver.switch_to.window(driver.window_handles[-1]) + driver.get('https://gotmyowndoma.in/') + driver.uninstall_addon(addon_id) + ext_path.unlink() + +@pytest.fixture() +def haketilo(driver): + addon_id = driver.install_addon(str(Path.cwd() / 'mozilla-build.zip'), + temporary=True) + + yield + + driver.uninstall_addon(addon_id) + +script_injector_script = '''\ +/* + * Selenium by default executes scripts in some weird one-time context. We want + * separately-loaded scripts to be able to access global variables defined + * before, including those declared with `const` or `let`. To achieve that, we + * run our scripts by injecting them into the page with a + + diff --git a/test/haketilo_test/data/pages/gotmyowndomain_https.html b/test/haketilo_test/data/pages/gotmyowndomain_https.html new file mode 100644 index 0000000..f602950 --- /dev/null +++ b/test/haketilo_test/data/pages/gotmyowndomain_https.html @@ -0,0 +1,35 @@ + + + + + + Schrodinger's Document + + + A nice, simple page for testing (using HTTPS). + + + diff --git a/test/haketilo_test/data/pages/scripts_to_block_1.html b/test/haketilo_test/data/pages/scripts_to_block_1.html new file mode 100644 index 0000000..164979d --- /dev/null +++ b/test/haketilo_test/data/pages/scripts_to_block_1.html @@ -0,0 +1,45 @@ + + + + + + + + + + + + + Click Meee! + + + + + + diff --git a/test/haketilo_test/default_profiles/icecat_empty/extensions.json b/test/haketilo_test/default_profiles/icecat_empty/extensions.json new file mode 100644 index 0000000..5f74ff3 --- /dev/null +++ b/test/haketilo_test/default_profiles/icecat_empty/extensions.json @@ -0,0 +1 @@ +{"schemaVersion":25,"addons":[{"id":"jid1-KtlZuoiikVfFew@jetpack","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/jid1-KtlZuoiikVfFew@jetpack"},{"id":"uBlock0@raymondhill.net","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/uBlock0@raymondhill.net.xpi"},{"id":"SubmitMe@0xbeef.coffee","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/SubmitMe@0xbeef.coffee"},{"id":"FreeUSPS@0xbeef.coffee","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/FreeUSPS@0xbeef.coffee"},{"id":"tortm-browser-button@jeremybenthum","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/tortm-browser-button@jeremybenthum"},{"id":"tprb.addon@searxes.danwin1210.me","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/tprb.addon@searxes.danwin1210.me"},{"id":"SimpleSumOfUs@0xbeef.coffee","location":"app-global","userDisabled":true,"path":"/usr/lib/icecat/browser/extensions/SimpleSumOfUs@0xbeef.coffee"}]} \ No newline at end of file diff --git a/test/haketilo_test/extension_crafting.py b/test/haketilo_test/extension_crafting.py new file mode 100644 index 0000000..97f5027 --- /dev/null +++ b/test/haketilo_test/extension_crafting.py @@ -0,0 +1,215 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +""" +Making temporary WebExtensions for use in the test suite +""" + +# This file is part of Haketilo. +# +# Copyright (C) 2021, 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this file's +# license. Although I request that you do not make use of this code in a +# proprietary program, I am not going to enforce this in court. + +import json +import zipfile +import re +import shutil +import subprocess + +from pathlib import Path +from uuid import uuid4 +from tempfile import TemporaryDirectory + +from selenium.webdriver.support.ui import WebDriverWait +from selenium.common.exceptions import NoSuchElementException + +from .misc_constants import * + +class ManifestTemplateValueToFill: + pass + +def manifest_template(): + return { + 'manifest_version': 2, + 'name': 'Haketilo test extension', + 'version': '1.0', + 'applications': { + 'gecko': { + 'id': ManifestTemplateValueToFill(), + 'strict_min_version': '60.0' + } + }, + 'permissions': [ + 'contextMenus', + 'webRequest', + 'webRequestBlocking', + 'activeTab', + 'notifications', + 'sessions', + 'storage', + 'tabs', + '', + 'unlimitedStorage' + ], + 'content_security_policy': "object-src 'none'; script-src 'self' https://serve.scrip.ts;", + 'web_accessible_resources': ['testpage.html'], + 'options_ui': { + 'page': 'testpage.html', + 'open_in_tab': True + }, + 'background': { + 'persistent': True, + 'scripts': ['__open_test_page.js', 'background.js'] + }, + 'content_scripts': [ + { + 'run_at': 'document_start', + 'matches': [''], + 'match_about_blank': True, + 'all_frames': True, + 'js': ['content.js'] + } + ] + } + +class ExtraHTML: + def __init__(self, html_path, append={}, wrap_into_htmldoc=True): + self.html_path = html_path + self.append = append + self.wrap_into_htmldoc = wrap_into_htmldoc + + def add_to_xpi(self, xpi, tmpdir=None): + if tmpdir is None: + with TemporaryDirectory() as tmpdir: + return self.add_to_xpi(xpi, tmpdir) + + append_flags = [] + for filename, code in self.append.items(): + append_flags.extend(['-A', f'{filename}:{code}']) + + awk = subprocess.run( + ['awk', '-f', awk_script_name, '--', *unit_test_defines, + *append_flags, '-H', self.html_path, '--write-js-deps', + '--output=files-to-copy', f'--output-dir={tmpdir}'], + stdout=subprocess.PIPE, cwd=proj_root, check=True + ) + + for path in filter(None, awk.stdout.decode().split('\n')): + xpi.write(proj_root / path, path) + + tmpdir = Path(tmpdir) + for path in tmpdir.rglob('*'): + relpath = str(path.relative_to(tmpdir)) + if not path.is_dir() and relpath != self.html_path: + xpi.write(path, relpath) + + with open(tmpdir / self.html_path, 'rt') as html_file: + html = html_file.read() + if self.wrap_into_htmldoc: + html = f'{html}' + xpi.writestr(self.html_path, html) + +default_background_script = '' +default_content_script = '' +default_test_page = ''' + + + + Extension's options page for testing + + +

Extension's options page for testing

+ + +''' + +open_test_page_script = '''(() => { +const page_url = browser.runtime.getURL("testpage.html"); +const execute_details = { + code: `window.wrappedJSObject.ext_page_url=${JSON.stringify(page_url)};` +}; +browser.tabs.query({currentWindow: true, active: true}) + .then(t => browser.tabs.executeScript(t.id, execute_details)); +})();''' + +def make_extension(destination_dir, + background_script=default_background_script, + content_script=default_content_script, + test_page=default_test_page, + extra_files={}, extra_html=[]): + if not hasattr(extra_html, '__iter__'): + extra_html = [extra_html] + manifest = manifest_template() + extension_id = '{%s}' % uuid4() + manifest['applications']['gecko']['id'] = extension_id + files = { + 'manifest.json' : json.dumps(manifest), + '__open_test_page.js': open_test_page_script, + 'background.js' : background_script, + 'content.js' : content_script, + 'testpage.html' : test_page, + **extra_files + } + destination_path = destination_dir / f'{extension_id}.xpi' + with zipfile.ZipFile(destination_path, 'x') as xpi: + for filename, contents in files.items(): + if hasattr(contents, '__call__'): + contents = contents() + xpi.writestr(filename, contents) + for html in extra_html: + html.add_to_xpi(xpi) + + return destination_path + +extract_base_url_re = re.compile(r'^(.*)manifest.json$') + +def get_extension_base_url(driver): + """ + Extension's internall UUID is not directly exposed in Selenium. Instead, we + can navigate to about:debugging and inspect the manifest URL present there + to get the base url like: + moz-extension://b225c78f-d108-4caa-8406-f38b37d8dee5/ + which can then be used to navigate to extension-bundled pages. + """ + # For newer Firefoxes + driver.get('about:debugging#/runtime/this-firefox') + + def get_manifest_link_newer_ff(driver): + try: + return driver.find_element_by_class_name('qa-manifest-url') + except NoSuchElementException: + pass + + try: + details = driver.find_element_by_class_name('error-page-details') + except NoSuchElementException: + return False + + if '#/runtime/this-firefox' in details.text: + return "not_newer_ff" + + manifest_link = WebDriverWait(driver, 10).until(get_manifest_link_newer_ff) + + if manifest_link == "not_newer_ff": + driver.get("about:debugging#addons") + driver.implicitly_wait(10) + manifest_link = driver.find_element_by_class_name('manifest-url') + driver.implicitly_wait(0) + + manifest_url = manifest_link.get_attribute('href') + return extract_base_url_re.match(manifest_url).group(1) diff --git a/test/haketilo_test/misc_constants.py b/test/haketilo_test/misc_constants.py new file mode 100644 index 0000000..9cac9dc --- /dev/null +++ b/test/haketilo_test/misc_constants.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +""" +Miscellaneous data that were found useful +""" + +# This file is part of Haketilo. +# +# Copyright (C) 2021 jahoti +# Copyright (C) 2021 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this +# file's license. Although I request that you do not make use of this code +# in a proprietary program, I am not going to enforce this in court. + +import re +from pathlib import Path + +here = Path(__file__).resolve().parent +proj_root = here.parent.parent +awk_script_name = 'compute_scripts.awk' + +unit_test_defines = ['-D', 'MOZILLA', '-D', 'MV2', '-D', 'TEST', + '-D', 'UNIT_TEST', '-D', 'DEBUG'] + +conf_line_regex = re.compile(r'^([^=]+) = (.*)$') +conf_settings = {} +with open(Path.cwd() / 'record.conf', 'rt') as conf: + for line in conf.readlines(): + match = conf_line_regex.match(line) + if match: + conf_settings[match.group(1).strip()] = match.group(2) + +default_proxy_host = '127.0.0.1' +default_proxy_port = 1337 + +default_cert_dir = proj_root / 'test' / 'certs' + +default_extension_uuid = 'a1291446-be95-48ad-a4c6-a475e389399b' +default_haketilo_id = '{6fe13369-88e9-440f-b837-5012fb3bedec}' + +mime_types = { + "7z": "application/x-7z-compressed", "oga": "audio/ogg", + "abw": "application/x-abiword", "ogv": "video/ogg", + "arc": "application/x-freearc", "ogx": "application/ogg", + "bin": "application/octet-stream", "opus": "audio/opus", + "bz": "application/x-bzip", "otf": "font/otf", + "bz2": "application/x-bzip2", "pdf": "application/pdf", + "css": "text/css", "png": "image/png", + "csv": "text/csv", "sh": "application/x-sh", + "gif": "image/gif", "svg": "image/svg+xml", + "gz": "application/gzip", "tar": "application/x-tar", + "htm": "text/html", "ts": "video/mp2t", + "html": "text/html", "ttf": "font/ttf", + "ico": "image/vnd.microsoft.icon", "txt": "text/plain", + "js": "text/javascript", "wav": "audio/wav", + "jpeg": "image/jpeg", "weba": "audio/webm", + "jpg": "image/jpeg", "webm": "video/webm", + "json": "application/json", "woff": "font/woff", + "mjs": "text/javascript", "woff2": "font/woff2", + "mp3": "audio/mpeg", "xhtml": "application/xhtml+xml", + "mp4": "video/mp4", "zip": "application/zip", + "mpeg": "video/mpeg", + "odp": "application/vnd.oasis.opendocument.presentation", + "ods": "application/vnd.oasis.opendocument.spreadsheet", + "odt": "application/vnd.oasis.opendocument.text", + "xml": "application/xml" # text/xml if readable from casual users +} diff --git a/test/haketilo_test/profiles.py b/test/haketilo_test/profiles.py new file mode 100755 index 0000000..ae997fc --- /dev/null +++ b/test/haketilo_test/profiles.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +""" +Browser profiles and Selenium driver initialization +""" + +# This file is part of Haketilo. +# +# Copyright (C) 2021 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this file's +# license. Although I request that you do not make use of this code in a +# proprietary program, I am not going to enforce this in court. + +from selenium import webdriver +from selenium.webdriver.firefox.options import Options +import json +from shutil import rmtree + +from .misc_constants import * + +class HaketiloFirefox(webdriver.Firefox): + """ + This wrapper class around selenium.webdriver.Firefox facilitates removing + the temporary profile directory after Firefox quits. + """ + def quit(self, *args, **kwargs): + profile_path = self.firefox_profile.path + super().quit(*args, **kwargs) + rmtree(profile_path, ignore_errors=True) + +def set_profile_proxy(profile, proxy_host, proxy_port): + """ + Create a Firefox profile that uses the specified HTTP proxy for all + protocols. + """ + # proxy type 1 designates "manual" + profile.set_preference('network.proxy.type', 1) + profile.set_preference('network.proxy.no_proxies_on', '') + profile.set_preference('network.proxy.share_proxy_settings', True) + + for proto in ['http', 'ftp', 'socks', 'ssl']: + profile.set_preference(f'network.proxy.{proto}', proxy_host) + profile.set_preference(f'network.proxy.{proto}_port', proxy_port) + profile.set_preference(f'network.proxy.backup.{proto}', '') + profile.set_preference(f'network.proxy.backup.{proto}_port', 0) + +def set_profile_csp_enabled(profile): + """ + By default, Firefox Driver disables CSP. The extension we're testing uses + CSP extensively, so we use this function to prepare a Firefox profile that + has it enabled. + """ + profile.set_preference('security.csp.enable', True) + +# The function below seems not to work for extensions that are +# temporarily-installed in Firefox safe mode. Testing is needed to see if it +# works with non-temporary extensions (without safe mode). +def set_webextension_uuid(profile, extension_id, uuid=default_extension_uuid): + """ + Firefox would normally assign a unique, random UUID to installed extension. + This UUID is needed to easily navigate to extension's settings page (and + other extension's pages). Since there's no way to learn such UUID with + current WebDriver implementation, this function works around this by telling + Firefox to use a predefined UUID for a certain extension. + """ + profile.set_preference('extensions.webextensions.uuids', + json.dumps({extension_id: uuid})) + +def firefox_safe_mode(firefox_binary=conf_settings['BROWSER_BINARY'], + proxy_host=default_proxy_host, + proxy_port=default_proxy_port): + """ + Initialize a Firefox instance controlled by selenium. The instance is + started in safe mode. + """ + profile = webdriver.FirefoxProfile() + set_profile_proxy(profile, proxy_host, proxy_port) + set_profile_csp_enabled(profile) + + options = Options() + options.add_argument('--safe-mode') + + return HaketiloFirefox(options=options, firefox_profile=profile, + firefox_binary=firefox_binary) + +def firefox_with_profile(firefox_binary=conf_settings['BROWSER_BINARY'], + profile_dir=conf_settings['CLEAN_PROFILE'], + proxy_host=default_proxy_host, + proxy_port=default_proxy_port): + """ + Initialize a Firefox instance controlled by selenium. The instance is + started using an empty profile (either the default one or the one passed to + `configure` script). The empty profile is meant to make Firefox start with + globally-installed extensions disabled. + """ + profile = webdriver.FirefoxProfile(profile_dir) + set_profile_proxy(profile, proxy_host, proxy_port) + set_profile_csp_enabled(profile) + set_webextension_uuid(profile, default_haketilo_id) + + return HaketiloFirefox(firefox_profile=profile, + firefox_binary=firefox_binary) diff --git a/test/haketilo_test/proxy_core.py b/test/haketilo_test/proxy_core.py new file mode 100644 index 0000000..f6bb820 --- /dev/null +++ b/test/haketilo_test/proxy_core.py @@ -0,0 +1,141 @@ +# SPDX-License-Identifier: BSD-3-Clause + +""" +The core for a "virtual network" proxy. +""" + +# This file is part of Haketilo. +# +# Copyright (c) 2015, inaz2 +# Copyright (C) 2021 jahoti +# Copyright (C) 2021 Wojtek Kosior +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of proxy2 nor the names of its contributors may be used to +# endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this file's +# license. Although I request that you do not make use of this code in a way +# incompliant with the license, I am not going to enforce this in court. + +from pathlib import Path +import socket, ssl, subprocess, sys, threading +from http.server import HTTPServer, BaseHTTPRequestHandler +from socketserver import ThreadingMixIn + +lock = threading.Lock() + +class ProxyRequestHandler(BaseHTTPRequestHandler): + """ + Handles a network request made to the proxy. Configures SSL encryption when + needed. + """ + def __init__(self, *args, **kwargs): + """ + Initialize self. Uses the same arguments as + http.server.BaseHTTPRequestHandler's constructor but also expect a + `certdir` keyword argument with appropriate path. + """ + self.certdir = Path(kwargs.pop('certdir')).resolve() + super().__init__(*args, **kwargs) + + def log_error(self, *args, **kwargs): + """ + Like log_error in http.server.BaseHTTPRequestHandler but suppresses + "Request timed out: timeout('timed out',)". + """ + if not isinstance(args[0], socket.timeout): + super().log_error(*args, **kwargs) + + def get_cert(self, hostname): + """ + If needed, generate a signed x509 certificate for `hostname`. Return + paths to certificate's key file and to certificate itself in a tuple. + """ + root_keyfile = self.certdir / 'rootCA.key' + root_certfile = self.certdir / 'rootCA.pem' + keyfile = self.certdir / 'site.key' + certfile = self.certdir / f'{hostname}.crt' + + with lock: + requestfile = self.certdir / f'{hostname}.csr' + if not certfile.exists(): + subprocess.run([ + 'openssl', 'req', '-new', '-key', str(keyfile), + '-subj', f'/CN={hostname}', '-out', str(requestfile) + ], check=True) + subprocess.run([ + 'openssl', 'x509', '-req', '-in', str(requestfile), + '-CA', str(root_certfile), '-CAkey', str(root_keyfile), + '-CAcreateserial', '-out', str(certfile), '-days', '1024' + ], check=True) + + return keyfile, certfile + + def do_CONNECT(self): + """Wrap the connection with SSL using on-demand signed certificate.""" + hostname = self.path.split(':')[0] + sslargs = {'server_side': True} + sslargs['keyfile'], sslargs['certfile'] = self.get_cert(hostname) + + self.send_response(200) + self.end_headers() + + self.connection = ssl.wrap_socket(self.connection, **sslargs) + self.rfile = self.connection.makefile('rb', self.rbufsize) + self.wfile = self.connection.makefile('wb', self.wbufsize) + + connection_header = self.headers.get('Proxy-Connection', '').lower() + self.close_connection = int(connection_header == 'close') + + def do_GET(self): + content_length = int(self.headers.get('Content-Length', 0)) + req_body = self.rfile.read(content_length) if content_length else None + + if self.path[0] == '/': + secure = 's' if isinstance(self.connection, ssl.SSLSocket) else '' + self.path = f'http{secure}://{self.headers["Host"]}{self.path}' + + self.handle_request(req_body) + + do_OPTIONS = do_DELETE = do_PUT = do_HEAD = do_POST = do_GET + + def handle_request(self, req_body): + """Default handler that does nothing. Please override.""" + pass + + +class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + """The actual proxy server""" + address_family, daemon_threads = socket.AF_INET6, True + + def handle_error(self, request, client_address): + """ + Like handle_error in http.server.HTTPServer but suppresses socket/ssl + related errors. + """ + cls, e = sys.exc_info()[:2] + if not (cls is socket.error or cls is ssl.SSLError): + return super().handle_error(request, client_address) diff --git a/test/haketilo_test/script_loader.py b/test/haketilo_test/script_loader.py new file mode 100644 index 0000000..66130bb --- /dev/null +++ b/test/haketilo_test/script_loader.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +""" +Loading of parts of Haketilo source for testing in browser +""" + +# This file is part of Haketilo. +# +# Copyright (C) 2021 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this file's +# license. Although I request that you do not make use of this code in a +# proprietary program, I am not going to enforce this in court. + +from pathlib import Path +import subprocess, re + +from .misc_constants import * + +def make_relative_path(path): + path = Path(path) + + if path.is_absolute(): + path = path.relative_to(proj_root) + + return path + +script_cache = {} + +def load_script(path, code_to_add=None): + """ + `path` is a .js file path in Haketilo sources. It may be absolute or + specified relative to Haketilo's project directory. `code_to_add` is + optional code to be appended to the end of the main file being imported. + it can contain directives like `#IMPORT`. + + Return a string containing script from `path` together with all other + scripts it depends on. Dependencies are wrapped in the same way Haketilo's + build system wraps them, with imports properly satisfied. The main script + being loaded is wrapped partially - it also has its imports satisfied, but + its code is executed in global scope instead of within an anonymous function + and imported variables are defined with `let` instead of `const` to allow + a dependency to be substituted by a mocked value. + """ + path = make_relative_path(path) + key = (str(path), code_to_add) + if key in script_cache: + return script_cache[key] + + append_flags = () if code_to_add is None else ('-A', ':'.join(key)) + + awk = subprocess.run(['awk', '-f', awk_script_name, '--', + *unit_test_defines, *append_flags, + '--output=amalgamate-js:' + str(path)], + stdout=subprocess.PIPE, cwd=proj_root, check=True) + script = awk.stdout.decode() + script_cache[key] = script + + return script diff --git a/test/haketilo_test/server.py b/test/haketilo_test/server.py new file mode 100755 index 0000000..7dc5e9e --- /dev/null +++ b/test/haketilo_test/server.py @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +""" +A modular "virtual network" proxy, +wrapping the classes in proxy_core.py +""" + +# This file is part of Haketilo. +# +# Copyright (C) 2021 jahoti +# Copyright (C) 2021 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this +# file's license. Although I request that you do not make use of this code +# in a proprietary program, I am not going to enforce this in court. + +from pathlib import Path +from urllib.parse import parse_qs +from threading import Thread +import traceback + +from .proxy_core import ProxyRequestHandler, ThreadingHTTPServer +from .misc_constants import * +from .world_wide_library import catalog as internet + +class RequestHijacker(ProxyRequestHandler): + def handle_request(self, req_body): + path_components = self.path.split('?', maxsplit=1) + path = path_components[0] + try: + # Response format: (status_code, headers (dict. of strings), + # body as bytes or filename containing body as string) + if path in internet: + info = internet[path] + if type(info) is tuple: + status_code, headers, body_file = info + resp_body = b'' + if body_file is not None: + if 'Content-Type' not in headers: + ext = body_file.suffix[1:] + if ext and ext in mime_types: + headers['Content-Type'] = mime_types[ext] + + with open(body_file, mode='rb') as f: + resp_body = f.read() + else: + # A function to evaluate to get the response + get_params, post_params = {}, {} + if len(path_components) == 2: + get_params = parse_qs(path_components[1]) + + # Parse POST parameters; currently only supports + # application/x-www-form-urlencoded + if req_body: + post_params = parse_qs(req_body.encode()) + + status_code, headers, resp_body = info(self.command, get_params, post_params) + if type(resp_body) == str: + resp_body = resp_body.encode() + + if type(status_code) != int or status_code <= 0: + raise Exception('Invalid status code %r' % status_code) + + for header, header_value in headers.items(): + if type(header) != str: + raise Exception('Invalid header key %r' % header) + + elif type(header_value) != str: + raise Exception('Invalid header value %r' % header_value) + else: + status_code, headers = 404, {'Content-Type': 'text/plain'} + resp_body = b'Handler for this URL not found.' + + except Exception: + status_code = 500 + headers = {'Content-Type': 'text/plain'} + resp_body = b'Internal Error:\n' + traceback.format_exc().encode() + + headers['Content-Length'] = str(len(resp_body)) + self.send_response(status_code) + for header, header_value in headers.items(): + self.send_header(header, header_value) + + self.end_headers() + if resp_body: + self.wfile.write(resp_body) + +def do_an_internet(certdir=Path.cwd() / 'certs', + port=default_proxy_port): + """Start up the proxy/server""" + class RequestHijackerWithCertdir(RequestHijacker): + def __init__(self, *args, **kwargs): + super().__init__(*args, certdir=certdir, **kwargs) + + httpd = ThreadingHTTPServer(('', port), RequestHijackerWithCertdir) + Thread(target=httpd.serve_forever).start() + + return httpd diff --git a/test/haketilo_test/test_integration.py b/test/haketilo_test/test_integration.py new file mode 100644 index 0000000..87d1827 --- /dev/null +++ b/test/haketilo_test/test_integration.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo integration tests +""" + +# This file is part of Haketilo +# +# Copyright (C) 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest + +from .extension_crafting import get_extension_base_url + +@pytest.mark.usefixtures('haketilo') +def test_integration(driver): + """ + Verify that the entire extension functions properly. Also verify bunlded + default settings get loaded properly. + """ + base_url = get_extension_base_url(driver) + driver.get(base_url + 'html/settings.html') + + for tab_head_id, item_text in [ + ('resources_head', 'Haketilo demonstrational script'), + ('mappings_head', 'Haketilo demonstrational message'), + ]: + driver.find_element_by_id(tab_head_id).click() + lst = driver.find_element_by_css_selector('.active_tab .item_list') + assert lst.is_displayed() + assert item_text in lst.text + + driver.find_element_by_id('repos_head').click() + lst = driver.find_element_by_css_selector('.active_tab .text_entries') + assert 'https://hydrilla.koszko.org/api_v1' in lst.text + + # TODO: do some more tests, including popup interaction and repository + # querying diff --git a/test/haketilo_test/unit/__init__.py b/test/haketilo_test/unit/__init__.py new file mode 100644 index 0000000..2b351bb --- /dev/null +++ b/test/haketilo_test/unit/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: CC0-1.0 +# Copyright (C) 2021 Wojtek Kosior diff --git a/test/haketilo_test/unit/test_CORS_bypass_server.py b/test/haketilo_test/unit/test_CORS_bypass_server.py new file mode 100644 index 0000000..45e4ebb --- /dev/null +++ b/test/haketilo_test/unit/test_CORS_bypass_server.py @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - routing HTTP requests through background script +""" + +# This file is part of Haketilo +# +# Copyright (C) 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +import json +from selenium.webdriver.support.ui import WebDriverWait + +from ..script_loader import load_script +from ..world_wide_library import some_data + +urls = { + 'resource': 'https://anotherdoma.in/resource/blocked/by/CORS.json', + 'nonexistent': 'https://nxdoma.in/resource.json', + 'invalid': 'w3csucks://invalid.url/' +} + +content_script = '''\ +const urls = %s; + +function fetch_data(url) { + return { + url, + to_get: ["ok", "status"], + to_call: ["text", "json"] + }; +} + +async function fetch_resources() { + const results = {}; + const promises = []; + for (const [name, url] of Object.entries(urls)) { + const sending = browser.runtime.sendMessage(["CORS_bypass", + fetch_data(url)]); + promises.push(sending.then(response => results[name] = response)); + } + + await Promise.all(promises); + + window.wrappedJSObject.haketilo_fetch_results = results; +} + +fetch_resources(); +''' + +content_script = content_script % json.dumps(urls); + +@pytest.mark.ext_data({ + 'content_script': content_script, + 'background_script': + lambda: load_script('background/CORS_bypass_server.js') + '; start();' +}) +@pytest.mark.usefixtures('webextension') +def test_CORS_bypass_server(driver, execute_in_page): + """ + Test if CORS bypassing works and if errors get properly forwarded. + """ + driver.get('https://gotmyowndoma.in/') + + # First, verify that requests without CORS bypass measures fail. + results = execute_in_page( + ''' + const result = {}; + let promises = []; + for (const [name, url] of Object.entries(arguments[0])) { + const [ok_cb, err_cb] = + ["ok", "err"].map(status => () => result[name] = status); + promises.push(fetch(url).then(ok_cb, err_cb)); + } + // Make the promises non-failing. + promises = promises.map(p => new Promise(cb => p.then(cb, cb))); + returnval(Promise.all(promises).then(() => result)); + ''', + {**urls, 'sameorigin': './nonexistent_resource'}) + + assert results == dict([*[(k, 'err') for k in urls.keys()], + ('sameorigin', 'ok')]) + + done = lambda d: d.execute_script('return window.haketilo_fetch_results;') + results = WebDriverWait(driver, 10).until(done) + + assert set(results['invalid'].keys()) == {'error'} + + assert set(results['nonexistent'].keys()) == \ + {'ok', 'status', 'text', 'error_json'} + assert results['nonexistent']['ok'] == False + assert results['nonexistent']['status'] == 404 + assert results['nonexistent']['text'] == 'Handler for this URL not found.' + + assert set(results['resource'].keys()) == {'ok', 'status', 'text', 'json'} + assert results['resource']['ok'] == True + assert results['resource']['status'] == 200 + assert results['resource']['text'] == some_data + assert results['resource']['json'] == json.loads(some_data) diff --git a/test/haketilo_test/unit/test_basic.py b/test/haketilo_test/unit/test_basic.py new file mode 100644 index 0000000..6ec54cc --- /dev/null +++ b/test/haketilo_test/unit/test_basic.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - base +""" + +# This file is part of Haketilo +# +# Copyright (C) 2021, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest + +from ..script_loader import load_script +from ..extension_crafting import ExtraHTML + +def test_driver(driver): + """ + A trivial test case that verifies mocked web pages served by proxy can be + accessed by the browser driven. + """ + for proto in ['http://', 'https://']: + driver.get(proto + 'gotmyowndoma.in') + title = driver.execute_script( + 'return document.getElementsByTagName("title")[0].innerText;' + ) + assert "Schrodinger's Document" in title + +@pytest.mark.get_page('https://gotmyowndoma.in') +def test_script_loader(execute_in_page): + """ + A trivial test case that verifies Haketilo's .js files can be properly + loaded into a test page together with their dependencies. + """ + execute_in_page(load_script('common/indexeddb.js')) + + assert 'mapping' in execute_in_page('returnval(stores.map(s => s[0]));') + +@pytest.mark.ext_data({}) +@pytest.mark.usefixtures('webextension') +def test_webextension(driver): + """ + A trivial test case that verifies a test WebExtension created and installed + by the `webextension` fixture works and redirects specially-constructed URLs + to its test page. + """ + heading = driver.execute_script( + 'return document.getElementsByTagName("h1")[0].innerText;' + ) + assert "Extension's options page for testing" in heading + +@pytest.mark.ext_data({ + 'extra_html': ExtraHTML( + 'html/default_blocking_policy.html', + { + 'html/default_blocking_policy.js': + 'document.body.innerHTML = `ski-ba-bop-ba ${typeof by_id}`;' + } + ), + 'navigate_to': 'html/default_blocking_policy.html' +}) +@pytest.mark.usefixtures('webextension') +def test_extra_html(driver): + """ + A trivial test case of the facility for loading the Haketilo's HTML files + into test WebExtension for unit-testing. + """ + assert driver.execute_script('return document.body.innerText') == \ + 'ski-ba-bop-ba function' diff --git a/test/haketilo_test/unit/test_broadcast.py b/test/haketilo_test/unit/test_broadcast.py new file mode 100644 index 0000000..7c2c051 --- /dev/null +++ b/test/haketilo_test/unit/test_broadcast.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - message broadcasting +""" + +# This file is part of Haketilo +# +# Copyright (C) 2021, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +from selenium.webdriver.support.ui import WebDriverWait + +from ..script_loader import load_script +from .utils import broker_js + +test_page_html = ''' + + +

d0 (channel `somebodyoncetoldme`)

+
+

d1 (channel `worldisgonnarollme`)

+
+

d2 (both channels)

+
+''' + +@pytest.mark.ext_data({ + 'background_script': broker_js, + 'test_page': test_page_html, + 'extra_files': { + 'testpage.js': lambda: load_script('common/broadcast.js') + } +}) +@pytest.mark.usefixtures('webextension') +def test_broadcast(driver, execute_in_page, wait_elem_text): + """ + A test that verifies the broadcasting system based on WebExtension messaging + API and implemented in `background/broadcast_broker.js` and + `common/broadcast.js` works correctly. + """ + # The broadcast facility is meant to enable message distribution between + # multiple contexts (e.g. different tabs/windows). Let's open the same + # extension's test page in a second window. + driver.execute_script( + ''' + window.open(window.location.href, "_blank"); + window.open(window.location.href, "_blank"); + ''') + WebDriverWait(driver, 10).until(lambda d: len(d.window_handles) == 3) + windows = [*driver.window_handles] + + # Let's first test if a simple message can be successfully broadcasted + driver.switch_to.window(windows[0]) + execute_in_page( + ''' + const divs = [0, 1, 2].map(n => document.getElementById("d" + n)); + let appender = n => (t => divs[n].append("\\n" + `[${t[0]}, ${t[1]}]`)); + let listener0 = listener_connection(appender(0)); + subscribe(listener0, "somebodyoncetoldme"); + ''') + + driver.switch_to.window(windows[1]) + execute_in_page( + ''' + let sender0 = sender_connection(); + out(sender0, "somebodyoncetoldme", "iaintthesharpesttool"); + ''') + + driver.switch_to.window(windows[0]) + wait_elem_text('d0', '[somebodyoncetoldme, iaintthesharpesttool]') + + # Let's add 2 more listeners + driver.switch_to.window(windows[0]) + execute_in_page( + ''' + let listener1 = listener_connection(appender(1)); + subscribe(listener1, "worldisgonnarollme"); + let listener2 = listener_connection(appender(2)); + subscribe(listener2, "worldisgonnarollme"); + subscribe(listener2, "somebodyoncetoldme"); + ''') + + # Let's send one message to one channel and one to the other. Verify they + # were received by the rght listeners. + driver.switch_to.window(windows[1]) + execute_in_page( + ''' + out(sender0, "somebodyoncetoldme", "intheshed"); + out(sender0, "worldisgonnarollme", "shewaslooking"); + ''') + + driver.switch_to.window(windows[0]) + wait_elem_text('d0', 'intheshed') + wait_elem_text('d1', 'shewaslooking') + wait_elem_text('d2', 'intheshed') + wait_elem_text('d2', 'shewaslooking') + + text = execute_in_page('returnval(divs[0].innerText);') + assert 'shewaslooking' not in text + text = execute_in_page('returnval(divs[1].innerText);') + assert 'intheshed' not in text + + # Let's create a second sender in third window and use it to send messages + # with the 'prepare' feature. + driver.switch_to.window(windows[2]) + execute_in_page( + ''' + let sender1 = sender_connection(); + prepare(sender1, "somebodyoncetoldme", "kindadumb"); + out(sender1, "worldisgonnarollme", "withherfinger"); + ''') + + driver.switch_to.window(windows[0]) + wait_elem_text('d1', 'withherfinger') + text = execute_in_page('returnval(divs[0].innerText);') + assert 'kindadumb' not in text + + driver.switch_to.window(windows[2]) + execute_in_page('flush(sender1);') + + driver.switch_to.window(windows[0]) + wait_elem_text('d0', 'kindadumb') + + # Let's verify that prepare()'d messages are properly discarded when + # discard() is called. + driver.switch_to.window(windows[2]) + execute_in_page( + ''' + prepare(sender1, "somebodyoncetoldme", "andherthumb"); + discard(sender1); + prepare(sender1, "somebodyoncetoldme", "andhermiddlefinger"); + flush(sender1); + ''') + + driver.switch_to.window(windows[0]) + wait_elem_text('d0', 'andhermiddlefinger') + text = execute_in_page('returnval(divs[0].innerText);') + assert 'andherthumb' not in text + + # Let's verify prepare()'d messages are properly auto-flushed when the other + # end of the connection gets killed (e.g. because browser tab gets closed). + driver.switch_to.window(windows[2]) + execute_in_page( + ''' + prepare(sender1, "worldisgonnarollme", "intheshape", 500); + ''') + driver.close() + + driver.switch_to.window(windows[0]) + wait_elem_text('d2', 'intheshape') + + # Verify listener's connection gets closed properly. + execute_in_page('close(listener0); close(listener1);') + + driver.switch_to.window(windows[1]) + execute_in_page('out(sender0, "worldisgonnarollme", "ofanL");') + execute_in_page('out(sender0, "somebodyoncetoldme", "forehead");') + + driver.switch_to.window(windows[0]) + wait_elem_text('d2', 'ofanL') + wait_elem_text('d2', 'forehead') + for i in (0, 1): + text = execute_in_page('returnval(divs[arguments[0]].innerText);', i) + assert 'ofanL' not in text + assert 'forehead' not in text diff --git a/test/haketilo_test/unit/test_content.py b/test/haketilo_test/unit/test_content.py new file mode 100644 index 0000000..8220160 --- /dev/null +++ b/test/haketilo_test/unit/test_content.py @@ -0,0 +1,190 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - main content script +""" + +# This file is part of Haketilo +# +# Copyright (C) 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +import json +from selenium.webdriver.support.ui import WebDriverWait + +from ..script_loader import load_script + +# From: +# https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/contentScripts/register +# it is unclear whether the dynamically-registered content script is guaranteed +# to be always executed after statically-registered ones. We want to test both +# cases, so we'll make the mocked dynamic content script execute before +# content.js on http:// pages and after it on https:// pages. +dynamic_script = \ + '''; + this.haketilo_secret = "abracadabra"; + this.haketilo_pattern_tree = {}; + this.haketilo_default_allow = false; + + if (this.haketilo_content_script_main) + this.haketilo_content_script_main(); + ''' + +content_script = \ + ''' + /* Mock dynamic content script - case 'before'. */ + if (/dynamic_before/.test(document.URL)) { + %s; + } + + /* Place amalgamated content.js here. */ + %s; + + /* Rest of mocks */ + + function mock_decide_policy() { + nonce = "12345"; + return { + allow: false, + mapping: "what-is-programmers-favorite-drinking-place", + payload: {identifier: "foo-bar"}, + nonce, + csp: "prefetch-src 'none'; script-src-attr 'none'; script-src 'nonce-12345'; script-src-elem 'nonce-12345';" + }; + } + + async function mock_payload_error([type, res_id]) { + if (type === "indexeddb_files") + return {error: {haketilo_error_type: "missing", id: res_id}}; + } + + async function mock_payload_ok([type, res_id]) { + if (type === "indexeddb_files") + return {files: [1, 2].map(n => `window.hak_injected_${n} = ${n};`)}; + } + + if (/payload_error/.test(document.URL)) { + browser.runtime.sendMessage = mock_payload_error; + decide_policy = mock_decide_policy; + } else if (/payload_ok/.test(document.URL)) { + browser.runtime.sendMessage = mock_payload_ok; + decide_policy = mock_decide_policy; + } + /* Otherwise, script blocking policy without payload to inject is used. */ + + const data_to_verify = {}; + function data_set(prop, val) { + data_to_verify[prop] = val; + window.wrappedJSObject.data_to_verify = JSON.stringify(data_to_verify); + } + + repo_query_cacher.start = () => data_set("cacher_started", true); + + enforce_blocking = policy => data_set("enforcing", policy); + + browser.runtime.onMessage.addListener = async function (listener_cb) { + await new Promise(cb => setTimeout(cb, 10)); + + /* Mock a good request. */ + const set_good = val => data_set("good_request_result", val); + data_set("good_request_returned", + !!listener_cb(["page_info"], {}, val => set_good(val))); + + /* Mock a bad request. */ + const set_bad = val => data_set("bad_request_result", val); + data_set("bad_request_returned", + !!listener_cb(["???"], {}, val => set_bad(val))); + } + + /* main() call - normally present in content.js, inside '#IF !UNIT_TEST'. */ + main(); + + /* Mock dynamic content script - case 'after'. */ + if (/#dynamic_after/.test(document.URL)) { + %s; + } + + data_set("script_run_without_errors", true); + ''' % (dynamic_script, load_script('content/content.js'), dynamic_script) + +@pytest.mark.ext_data({'content_script': content_script}) +@pytest.mark.usefixtures('webextension') +@pytest.mark.parametrize('target1', ['dynamic_before'])#, 'dynamic_after']) +@pytest.mark.parametrize('target2', [ + 'scripts_blocked', + 'payload_error', + 'payload_ok' +]) +def test_content_unprivileged_page(driver, execute_in_page, target1, target2): + """ + Test functioning of content.js on an page using unprivileged schema (e.g. + 'https://' and not 'about:'). + """ + driver.get(f'https://gotmyowndoma.in/index.html#{target1}-{target2}') + + def get_data(driver): + data = driver.execute_script('return window.data_to_verify;') + return data if 'good_request_result' in data else False + + data = json.loads(WebDriverWait(driver, 10).until(get_data)) + + assert 'gotmyowndoma.in' in data['good_request_result']['url'] + assert 'bad_request_result' not in data + + assert data['good_request_returned'] == True + assert data['bad_request_returned'] == False + + assert data['cacher_started'] == True + + for obj in (data['good_request_result'], data['enforcing']): + assert obj['allow'] == False + + assert 'error' not in data['enforcing'] + + if target2.startswith('payload'): + for obj in (data['good_request_result'], data['enforcing']): + assert obj['payload']['identifier'] == 'foo-bar' + assert 'mapping' in obj + else: + assert 'payload' not in data['enforcing'] + assert 'mapping' not in data['enforcing'] + + assert data['script_run_without_errors'] == True + + def vars_made_by_payload(driver): + vars_values = driver.execute_script( + 'return [1, 2].map(n => window[`hak_injected_${n}`]);' + ) + if vars_values != [None, None]: + return vars_values + + if target2 == 'payload_error': + assert data['good_request_result']['error'] == { + 'haketilo_error_type': 'missing', + 'id': 'foo-bar' + } + elif target2 == 'payload_ok': + vars_values = WebDriverWait(driver, 10).until(vars_made_by_payload) + assert vars_values == [1, 2] + +@pytest.mark.ext_data({'content_script': content_script}) +@pytest.mark.usefixtures('webextension') +@pytest.mark.parametrize('target', ['dynamic_before', 'dynamic_after']) +def test_content_privileged_page(driver, execute_in_page, target): + """ + Test functioning of content.js on an page considered privileged (e.g. a + directory listing at 'file:///'). + """ + driver.get(f'file:///#{target}') + data = json.loads(driver.execute_script('return window.data_to_verify;')) + + assert data == {'script_run_without_errors': True} diff --git a/test/haketilo_test/unit/test_default_policy_dialog.py b/test/haketilo_test/unit/test_default_policy_dialog.py new file mode 100644 index 0000000..a1c825f --- /dev/null +++ b/test/haketilo_test/unit/test_default_policy_dialog.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - default script blocking policy dialog +""" + +# This file is part of Haketilo +# +# Copyright (C) 2022, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest + +from ..extension_crafting import ExtraHTML +from ..script_loader import load_script +from .utils import broker_js + +@pytest.mark.ext_data({ + 'background_script': broker_js, + 'extra_html': ExtraHTML( + 'html/default_blocking_policy.html', + { + 'html/default_blocking_policy.js': + 'init_default_policy_dialog();' + } + ), + 'navigate_to': 'html/default_blocking_policy.html' +}) +@pytest.mark.usefixtures('webextension') +def test_default_blocking_policy_dialog(driver, wait_elem_text): + """ + A test case for the dialog that facilitates toggling the default policy of + script blocking. + """ + wait_elem_text('current_policy_span', 'block') + + driver.find_element_by_id('toggle_policy_but').click() + wait_elem_text('current_policy_span', 'allow') + + driver.find_element_by_id('toggle_policy_but').click() + wait_elem_text('current_policy_span', 'block') diff --git a/test/haketilo_test/unit/test_dialog.py b/test/haketilo_test/unit/test_dialog.py new file mode 100644 index 0000000..63af79e --- /dev/null +++ b/test/haketilo_test/unit/test_dialog.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - showing an error/info/question dalog +""" + +# This file is part of Haketilo +# +# Copyright (C) 2022, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest + +from ..extension_crafting import ExtraHTML +from ..script_loader import load_script + +@pytest.mark.ext_data({ + 'extra_html': ExtraHTML('html/dialog.html', {}), + 'navigate_to': 'html/dialog.html' +}) +@pytest.mark.usefixtures('webextension') +def test_dialog_show_close(driver, execute_in_page): + """ + A test case of basic dialog showing/closing. + """ + execute_in_page(load_script('html/dialog.js')) + buts = execute_in_page( + ''' + let cb_calls, call_prom; + const dialog_context = make(() => cb_calls.push("show"), + () => cb_calls.push("hide")); + document.body.append(dialog_context.main_div); + const buts = {}; + for (const but of document.getElementsByTagName("button")) + buts[but.textContent] = but; + returnval(buts); + ''') + + for i, (dialog_function, but_text, hidden, expected_result) in enumerate([ + ('info', 'Ok', ['Yes', 'No'], None), + ('error', 'Ok', ['Yes', 'No'], None), + ('error', None, ['Yes', 'No'], None), + ('loader', None, ['Yes', 'No', 'Ok'], None), + ('ask', 'Yes', ['Ok'], True), + ('ask', None, ['Ok'], None), + ('ask', 'No', ['Ok'], False) + ]): + cb_calls, is_shown = execute_in_page( + f''' + cb_calls = []; + call_prom = {dialog_function}(dialog_context, + `sample_text_${{arguments[0]}}`); + returnval([cb_calls, dialog_context.shown]); + ''', + i) + assert cb_calls == ['show'] + assert is_shown == True + + page_source = driver.page_source + assert f'sample_text_{i}' in page_source + assert f'sample_text_{i - 1}' not in page_source + + # Verify the right buttons are displayed. + for text, but in buts.items(): + if text in hidden: + assert not but.is_displayed() + # Verify clicking a hidden button does nothing. + execute_in_page('buts[arguments[0]].click();', text) + assert execute_in_page('returnval(cb_calls);') == cb_calls + else: + assert but.is_displayed() + + if but_text is None: + execute_in_page('close_dialog(dialog_context);') + else: + buts[but_text].click() + + cb_calls, result, is_shown = execute_in_page( + '''{ + const values_cb = r => [cb_calls, r, dialog_context.shown]; + returnval(call_prom.then(values_cb)); + }''') + assert cb_calls == ['show', 'hide'] + assert result == expected_result + assert is_shown == False + +@pytest.mark.ext_data({ + 'extra_html': ExtraHTML('html/dialog.html', {}), + 'navigate_to': 'html/dialog.html' +}) +@pytest.mark.usefixtures('webextension') +def test_dialog_queue(driver, execute_in_page): + """ + A test case of queuing dialog display operations. + """ + execute_in_page(load_script('html/dialog.js')) + execute_in_page( + ''' + let cb_calls = [], call_proms = []; + const dialog_context = make(() => cb_calls.push("show"), + () => cb_calls.push("hide")); + document.body.append(dialog_context.main_div); + ''') + + buts = driver.find_elements_by_tag_name('button') + buts = dict([(but.text, but) for but in buts]) + + for i in range(5): + cb_calls, is_shown, msg_elem = execute_in_page( + ''' + call_proms.push(ask(dialog_context, "somequestion" + arguments[0])); + returnval([cb_calls, dialog_context.shown, dialog_context.msg]); + ''', + i) + assert cb_calls == ['show'] + assert is_shown == True + assert msg_elem.text == 'somequestion0' + + for i in range(5): + buts['Yes' if i & 1 else 'No'].click() + cb_calls, is_shown, msg_elem, result = execute_in_page( + '''{ + const values_cb = + r => [cb_calls, dialog_context.shown, dialog_context.msg, r]; + returnval(call_proms.splice(0, 1)[0].then(values_cb)); + }''') + if i < 4: + assert cb_calls == ['show'] + assert is_shown == True + assert msg_elem.text == f'somequestion{i + 1}' + else: + assert cb_calls == ['show', 'hide'] + assert is_shown == False + + assert result == bool(i & 1) diff --git a/test/haketilo_test/unit/test_indexeddb.py b/test/haketilo_test/unit/test_indexeddb.py new file mode 100644 index 0000000..c2d5427 --- /dev/null +++ b/test/haketilo_test/unit/test_indexeddb.py @@ -0,0 +1,490 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - IndexedDB access +""" + +# This file is part of Haketilo +# +# Copyright (C) 2021, 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +import json +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.common.exceptions import WebDriverException + +from ..script_loader import load_script +from .utils import * + +# Sample resource definitions. They'd normally contain more fields but here we +# use simplified versions. + +def make_sample_resource(): + return { + 'source_copyright': [ + sample_file_ref('report.spdx'), + sample_file_ref('LICENSES/somelicense.txt') + ], + 'type': 'resource', + 'identifier': 'helloapple', + 'scripts': [sample_file_ref('hello.js'), sample_file_ref('bye.js')] + } + +def make_sample_mapping(): + return { + 'source_copyright': [ + sample_file_ref('report.spdx'), + sample_file_ref('README.md') + ], + 'type': 'mapping', + 'identifier': 'helloapple' + } + +@pytest.mark.get_page('https://gotmyowndoma.in') +def test_haketilodb_item_modifications(driver, execute_in_page): + """ + indexeddb.js facilitates operating on Haketilo's internal database. + Verify database operations on mappings/resources work properly. + """ + execute_in_page(load_script('common/indexeddb.js')) + mock_broadcast(execute_in_page) + + # Start with no database. + clear_indexeddb(execute_in_page) + + sample_item = make_sample_resource() + sample_item['source_copyright'][0]['extra_prop'] = True + + execute_in_page( + '''{ + const promise = start_items_transaction(["resource"], arguments[1]) + .then(ctx => save_item(arguments[0], ctx).then(() => ctx)) + .then(finalize_transaction); + returnval(promise); + }''', + sample_item, {'sha256': sample_files_by_sha256}) + + database_contents = get_db_contents(execute_in_page) + + assert len(database_contents['file']) == 4 + assert all([sample_files_by_sha256[file['sha256']] == file['contents'] + for file in database_contents['file']]) + assert all([len(file) == 2 for file in database_contents['file']]) + + assert len(database_contents['file_uses']) == 4 + assert all([uses['uses'] == 1 for uses in database_contents['file_uses']]) + assert set([uses['sha256'] for uses in database_contents['file_uses']]) \ + == set([file['sha256'] for file in database_contents['file']]) + + assert database_contents['mapping'] == [] + assert database_contents['resource'] == [sample_item] + + # See if trying to add an item without providing all its files ends in an + # exception and aborts the transaction as it should. + sample_item['scripts'].append(sample_file_ref('combined.js')) + incomplete_files = {**sample_files_by_sha256} + incomplete_files.pop(sample_files['combined.js']['sha256']) + exception = execute_in_page( + '''{ + const args = arguments; + async function try_add_item() + { + const context = + await start_items_transaction(["resource"], args[1]); + try { + await save_item(args[0], context); + await finalize_transaction(context); + return; + } catch(e) { + return e; + } + } + returnval(try_add_item()); + }''', + sample_item, {'sha256': incomplete_files}) + + previous_database_contents = database_contents + database_contents = get_db_contents(execute_in_page) + + assert 'file not present' in exception + for key, val in database_contents.items(): + keyfun = lambda item: item.get('sha256') or item['identifier'] + assert sorted(previous_database_contents[key], key=keyfun) \ + == sorted(val, key=keyfun) + + # See if adding another item that partially uses first's files works OK. + sample_item = make_sample_mapping() + database_contents = execute_in_page( + '''{ + const promise = start_items_transaction(["mapping"], arguments[1]) + .then(ctx => save_item(arguments[0], ctx).then(() => ctx)) + .then(finalize_transaction); + returnval(promise); + }''', + sample_item, {'sha256': sample_files_by_sha256}) + + database_contents = get_db_contents(execute_in_page) + + names = ['README.md', 'report.spdx', 'LICENSES/somelicense.txt', 'hello.js', + 'bye.js'] + sample_files_list = [sample_files[name] for name in names] + uses_list = [1, 2, 1, 1, 1] + + uses = dict([(uses['sha256'], uses['uses']) + for uses in database_contents['file_uses']]) + assert uses == dict([(file['sha256'], nr) + for file, nr in zip(sample_files_list, uses_list)]) + + files = dict([(file['sha256'], file['contents']) + for file in database_contents['file']]) + assert files == dict([(file['sha256'], file['contents']) + for file in sample_files_list]) + + del database_contents['resource'][0]['source_copyright'][0]['extra_prop'] + assert database_contents['resource'] == [make_sample_resource()] + assert database_contents['mapping'] == [sample_item] + + # Try removing the items to get an empty database again. + results = [None, None] + for i, item_type in enumerate(['resource', 'mapping']): + execute_in_page( + f'''{{ + const remover = remove_{item_type}; + const promise = + start_items_transaction(["{item_type}"], {{}}) + .then(ctx => remover('helloapple', ctx).then(() => ctx)) + .then(finalize_transaction); + returnval(promise); + }}''') + + results[i] = get_db_contents(execute_in_page) + + names = ['README.md', 'report.spdx'] + sample_files_list = [sample_files[name] for name in names] + uses_list = [1, 1] + + uses = dict([(uses['sha256'], uses['uses']) + for uses in results[0]['file_uses']]) + assert uses == dict([(file['sha256'], 1) for file in sample_files_list]) + + files = dict([(file['sha256'], file['contents']) + for file in results[0]['file']]) + assert files == dict([(file['sha256'], file['contents']) + for file in sample_files_list]) + + assert results[0]['resource'] == [] + assert results[0]['mapping'] == [sample_item] + + assert results[1] == dict([(key, []) for key in results[0].keys()]) + + # Try initializing an empty database with sample initial data object. + sample_resource = make_sample_resource() + sample_mapping = make_sample_mapping() + initial_data = { + 'resource': { + 'helloapple': { + '1.12': sample_resource, + '0.9': 'something_that_should_get_ignored', + '1': 'something_that_should_get_ignored', + '1.1': 'something_that_should_get_ignored', + '1.11.1': 'something_that_should_get_ignored', + } + }, + 'mapping': { + 'helloapple': { + '0.1.1': sample_mapping + } + }, + 'file': { + 'sha256': sample_files_by_sha256 + } + } + + clear_indexeddb(execute_in_page) + execute_in_page('initial_data = arguments[0];', initial_data) + database_contents = get_db_contents(execute_in_page) + + assert database_contents['resource'] == [sample_resource] + assert database_contents['mapping'] == [sample_mapping] + +@pytest.mark.get_page('https://gotmyowndoma.in') +def test_haketilodb_settings(driver, execute_in_page): + """ + indexeddb.js facilitates operating on Haketilo's internal database. + Verify assigning/retrieving values of simple "setting" item works properly. + """ + execute_in_page(load_script('common/indexeddb.js')) + mock_broadcast(execute_in_page) + + # Start with no database. + clear_indexeddb(execute_in_page) + + assert get_db_contents(execute_in_page)['setting'] == [] + + assert execute_in_page('returnval(get_setting("option15"));') == None + + execute_in_page('returnval(set_setting("option15", "disable"));') + assert execute_in_page('returnval(get_setting("option15"));') == 'disable' + + execute_in_page('returnval(set_setting("option15", "enable"));') + assert execute_in_page('returnval(get_setting("option15"));') == 'enable' + +@pytest.mark.get_page('https://gotmyowndoma.in') +def test_haketilodb_allowing(driver, execute_in_page): + """ + indexeddb.js facilitates operating on Haketilo's internal database. + Verify changing the "blocking" configuration for a URL works properly. + """ + execute_in_page(load_script('common/indexeddb.js')) + mock_broadcast(execute_in_page) + + # Start with no database. + clear_indexeddb(execute_in_page) + + assert get_db_contents(execute_in_page)['blocking'] == [] + + def run_with_sample_url(expr): + return execute_in_page(f'returnval({expr});', 'https://example.com/**') + + assert None == run_with_sample_url('get_allowing(arguments[0])') + + run_with_sample_url('set_disallowed(arguments[0])') + assert False == run_with_sample_url('get_allowing(arguments[0])') + + run_with_sample_url('set_allowed(arguments[0])') + assert True == run_with_sample_url('get_allowing(arguments[0])') + + run_with_sample_url('set_default_allowing(arguments[0])') + assert None == run_with_sample_url('get_allowing(arguments[0])') + +@pytest.mark.get_page('https://gotmyowndoma.in') +def test_haketilodb_repos(driver, execute_in_page): + """ + indexeddb.js facilitates operating on Haketilo's internal database. + Verify operations on repositories list work properly. + """ + execute_in_page(load_script('common/indexeddb.js')) + mock_broadcast(execute_in_page) + + # Start with no database. + clear_indexeddb(execute_in_page) + + assert get_db_contents(execute_in_page)['repo'] == [] + + sample_urls = ['https://hdrlla.example.com/', 'https://hdrlla.example.org'] + + assert [] == execute_in_page('returnval(get_repos());') + + execute_in_page('returnval(set_repo(arguments[0]));', sample_urls[0]) + assert [sample_urls[0]] == execute_in_page('returnval(get_repos());') + + execute_in_page('returnval(set_repo(arguments[0]));', sample_urls[1]) + assert set(sample_urls) == set(execute_in_page('returnval(get_repos());')) + + execute_in_page('returnval(del_repo(arguments[0]));', sample_urls[0]) + assert [sample_urls[1]] == execute_in_page('returnval(get_repos());') + +test_page_html = ''' + + + + +''' + +@pytest.mark.ext_data({ + 'background_script': broker_js, + 'test_page': test_page_html, + 'extra_files': { + 'testpage.js': lambda: load_script('common/indexeddb.js') + } +}) +@pytest.mark.usefixtures('webextension') +def test_haketilodb_track(driver, execute_in_page, wait_elem_text): + """ + Verify IndexedDB object change notifications are correctly broadcasted + through extension's background script and allow for object store contents + to be tracked in any execution context. + """ + # Let's open the same extension's test page in a second window. Window 1 + # will be used to make changes to IndexedDB and window 0 to "track" those + # changes. + driver.execute_script('window.open(window.location.href, "_blank");') + WebDriverWait(driver, 10).until(lambda d: len(d.window_handles) == 2) + windows = [*driver.window_handles] + + # Create elements that will have tracked data inserted under them. + driver.switch_to.window(windows[0]) + execute_in_page( + ''' + for (const store_name of trackable) { + const h2 = document.createElement("h2"); + h2.innerText = store_name; + document.body.append(h2); + + const ul = document.createElement("ul"); + ul.id = store_name; + document.body.append(ul); + } + ''') + + # Mock initial_data. + sample_resource = make_sample_resource() + sample_mapping = make_sample_mapping() + initial_data = { + 'resource': { + 'helloapple': { + '1.0': sample_resource + } + }, + 'mapping': { + 'helloapple': { + '0.1.1': sample_mapping + } + }, + 'file': { + 'sha256': sample_files_by_sha256 + } + } + driver.switch_to.window(windows[1]) + execute_in_page('initial_data = arguments[0];', initial_data) + execute_in_page('returnval(set_setting("option15", "123"));') + execute_in_page('returnval(set_repo("https://hydril.la"));') + execute_in_page('returnval(set_disallowed("file:///*"));') + + # See if track.*() functions properly return the already-existing items. + driver.switch_to.window(windows[0]) + execute_in_page( + ''' + function update_item(store_name, change) + { + const elem_id = `${store_name}_${change.key}`; + let elem = document.getElementById(elem_id); + elem = elem || document.createElement("li"); + elem.id = elem_id; + elem.innerText = JSON.stringify(change.new_val); + document.getElementById(store_name).append(elem); + if (change.new_val === undefined) + elem.remove(); + } + + let resource_tracking, resource_items, mapping_tracking, mapping_items; + + async function start_reporting() + { + const props = new Map(stores.map(([sn, opt]) => [sn, opt.keyPath])); + for (const store_name of trackable) { + [tracking, items] = + await track[store_name](ch => update_item(store_name, ch)); + const prop = props.get(store_name); + for (const item of items) + update_item(store_name, {key: item[prop], new_val: item}); + } + } + + returnval(start_reporting()); + ''') + + item_counts = execute_in_page( + '''{ + const childcount = id => document.getElementById(id).childElementCount; + returnval(trackable.map(childcount)); + }''') + assert item_counts == [1 for _ in item_counts] + for elem_id, json_value in [ + ('resource_helloapple', sample_resource), + ('mapping_helloapple', sample_mapping), + ('setting_option15', {'name': 'option15', 'value': '123'}), + ('repo_https://hydril.la', {'url': 'https://hydril.la'}), + ('blocking_file:///*', {'pattern': 'file:///*', 'allow': False}) + ]: + assert json.loads(driver.find_element_by_id(elem_id).text) == json_value + + # See if item additions get tracked properly. + driver.switch_to.window(windows[1]) + sample_resource2 = make_sample_resource() + sample_resource2['identifier'] = 'helloapple-copy' + sample_mapping2 = make_sample_mapping() + sample_mapping2['identifier'] = 'helloapple-copy' + sample_data = { + 'resource': { + 'helloapple-copy': { + '1.0': sample_resource2 + } + }, + 'mapping': { + 'helloapple-copy': { + '0.1.1': sample_mapping2 + } + }, + 'file': { + 'sha256': sample_files_by_sha256 + }, + 'repo': [ + 'https://hydril2.la/' + ] + } + execute_in_page('returnval(save_items(arguments[0]));', sample_data) + execute_in_page('returnval(set_setting("option22", "abc"));') + execute_in_page('returnval(set_repo("https://hydril3.la/"));') + execute_in_page('returnval(set_allowed("ftp://a.bc/"));') + + driver.switch_to.window(windows[0]) + driver.implicitly_wait(10) + for elem_id, json_value in [ + ('resource_helloapple-copy', sample_resource2), + ('mapping_helloapple-copy', sample_mapping2), + ('setting_option22', {'name': 'option22', 'value': 'abc'}), + ('repo_https://hydril2.la/', {'url': 'https://hydril2.la/'}), + ('repo_https://hydril3.la/', {'url': 'https://hydril3.la/'}), + ('blocking_ftp://a.bc/', {'pattern': 'ftp://a.bc/', 'allow': True}) + ]: + assert json.loads(driver.find_element_by_id(elem_id).text) == json_value + driver.implicitly_wait(0) + + # See if item deletions/modifications get tracked properly. + driver.switch_to.window(windows[1]) + execute_in_page( + '''{ + async function change_remove_items() + { + const store_names = ["resource", "mapping"]; + const ctx = await start_items_transaction(store_names, {}); + await remove_resource("helloapple", ctx); + await remove_mapping("helloapple-copy", ctx); + await finalize_transaction(ctx); + await set_setting("option22", null); + await del_repo("https://hydril.la"); + await set_default_allowing("file:///*"); + await set_disallowed("ftp://a.bc/"); + } + returnval(change_remove_items()); + }''') + + removed_ids = ['mapping_helloapple-copy', 'resource_helloapple', + 'repo_https://hydril.la', 'blocking_file:///*'] + def condition_items_absent_and_changed(driver): + for id in removed_ids: + try: + driver.find_element_by_id(id) + return False + except WebDriverException: + pass + + option_text = driver.find_element_by_id('setting_option22').text + blocking_text = driver.find_element_by_id('blocking_ftp://a.bc/').text + return (json.loads(option_text)['value'] == None and + json.loads(blocking_text)['allow'] == False) + + driver.switch_to.window(windows[0]) + WebDriverWait(driver, 10).until(condition_items_absent_and_changed) diff --git a/test/haketilo_test/unit/test_indexeddb_files_server.py b/test/haketilo_test/unit/test_indexeddb_files_server.py new file mode 100644 index 0000000..6ddfba8 --- /dev/null +++ b/test/haketilo_test/unit/test_indexeddb_files_server.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - serving indexeddb resource script files to content scripts +""" + +# This file is part of Haketilo +# +# Copyright (C) 2021,2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +import copy +from uuid import uuid4 +from selenium.webdriver.support.ui import WebDriverWait + +from ..script_loader import load_script +from .utils import * + +""" +How many test resources we're going to have. +""" +count = 15 + +sample_files_list = [(f'file_{n}_{i}', f'contents {n} {i}') + for n in range(count) for i in range(2)] + +sample_files = dict(sample_files_list) + +sample_files, sample_files_by_sha256 = make_sample_files(sample_files) + +def make_sample_resource_with_deps(n): + resource = make_sample_resource(with_files=False) + + resource['identifier'] = f'res-{n}' + resource['dependencies'] = [{'identifier': f'res-{m}'} + for m in range(max(n - 4, 0), n)] + resource['scripts'] = [sample_file_ref(f'file_{n}_{i}', sample_files) + for i in range(2)] + + return resource + +resources = [make_sample_resource_with_deps(n) for n in range(count)] + +sample_data = { + 'resource': sample_data_dict(resources), + 'mapping': {}, + 'file': { + 'sha256': sample_files_by_sha256 + } +} + +def prepare_test_page(initial_indexeddb_data, execute_in_page): + js = load_script('background/indexeddb_files_server.js', + code_to_add='#IMPORT common/broadcast.js') + execute_in_page(js) + + mock_broadcast(execute_in_page) + clear_indexeddb(execute_in_page) + + execute_in_page( + ''' + let registered_listener; + const new_addListener = cb => registered_listener = cb; + + browser = {runtime: {onMessage: {addListener: new_addListener}}}; + + haketilodb.save_items(arguments[0]); + + start(); + ''', + initial_indexeddb_data) + +@pytest.mark.get_page('https://gotmyowndoma.in') +def test_indexeddb_files_server_normal_usage(driver, execute_in_page): + """ + Test querying resource files (with resource dependency resolution) + from IndexedDB and serving them in messages to content scripts. + """ + prepare_test_page(sample_data, execute_in_page) + + # Verify other types of messages are ignored. + function_returned_value = execute_in_page( + ''' + returnval(registered_listener(["???"], {}, + () => location.reload())); + ''') + assert function_returned_value == None + + # Verify single resource's files get properly resolved. + function_returned_value = execute_in_page( + ''' + var result_cb, contents_prom = new Promise(cb => result_cb = cb); + + returnval(registered_listener(["indexeddb_files", "res-0"], + {}, result_cb)); + ''') + assert function_returned_value == True + + assert execute_in_page('returnval(contents_prom);') == \ + {'files': [tuple[1] for tuple in sample_files_list[0:2]]} + + # Verify multiple resources' files get properly resolved. + function_returned_value = execute_in_page( + ''' + var result_cb, contents_prom = new Promise(cb => result_cb = cb); + + returnval(registered_listener(["indexeddb_files", arguments[0]], + {}, result_cb)); + ''', + f'res-{count - 1}') + assert function_returned_value == True + + assert execute_in_page('returnval(contents_prom);') == \ + {'files': [tuple[1] for tuple in sample_files_list]} + +@pytest.mark.get_page('https://gotmyowndoma.in') +@pytest.mark.parametrize('error', [ + 'missing', + 'circular', + 'db', + 'other' +]) +def test_indexeddb_files_server_errors(driver, execute_in_page, error): + """ + Test reporting of errors when querying resource files (with resource + dependency resolution) from IndexedDB and serving them in messages to + content scripts. + """ + sample_data_copy = copy.deepcopy(sample_data) + + if error == 'missing': + del sample_data_copy['resource']['res-3'] + elif error == 'circular': + res3_defs = sample_data_copy['resource']['res-3'].values() + next(iter(res3_defs))['dependencies'].append({'identifier': 'res-8'}) + + prepare_test_page(sample_data_copy, execute_in_page) + + if error == 'db': + execute_in_page('haketilodb.idb_get = t => t.onerror("oooops");') + elif error == 'other': + execute_in_page('haketilodb.idb_get = () => {throw "oooops"};') + + response = execute_in_page( + ''' + var result_cb, contents_prom = new Promise(cb => result_cb = cb); + + registered_listener(["indexeddb_files", arguments[0]], + {}, result_cb); + + returnval(contents_prom); + ''', + f'res-{count - 1}') + + assert response['error']['haketilo_error_type'] == error + + if error == 'missing': + assert response['error']['id'] == 'res-3' + elif error == 'circular': + assert response['error']['id'] in ('res-3', 'res-8') + elif error not in ('db', 'other'): + raise Exception('made a typo in test function params?') diff --git a/test/haketilo_test/unit/test_install.py b/test/haketilo_test/unit/test_install.py new file mode 100644 index 0000000..f4bc483 --- /dev/null +++ b/test/haketilo_test/unit/test_install.py @@ -0,0 +1,423 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - item installation dialog +""" + +# This file is part of Haketilo +# +# Copyright (C) 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +import json +from selenium.webdriver.support.ui import WebDriverWait + +from ..extension_crafting import ExtraHTML +from ..script_loader import load_script +from .utils import * + +def setup_view(driver, execute_in_page): + mock_cacher(execute_in_page) + + execute_in_page(load_script('html/install.js')) + container_ids, containers_objects = execute_in_page( + ''' + const cb_calls = []; + const install_view = new InstallView(0, + () => cb_calls.push("show"), + () => cb_calls.push("hide")); + document.body.append(install_view.main_div); + const ets = () => install_view.item_entries; + const shw = slice => [cb_calls.slice(slice || 0), install_view.shown]; + returnval([container_ids, container_ids.map(cid => install_view[cid])]); + ''') + + containers = dict(zip(container_ids, containers_objects)) + + def assert_container_displayed(container_id): + for cid, cobj in zip(container_ids, containers_objects): + assert (cid == container_id) == cobj.is_displayed() + + return containers, assert_container_displayed + +install_ext_data = { + 'background_script': broker_js, + 'extra_html': ExtraHTML('html/install.html', {}), + 'navigate_to': 'html/install.html' +} + +@pytest.mark.ext_data(install_ext_data) +@pytest.mark.usefixtures('webextension') +@pytest.mark.parametrize('complex_variant', [False, True]) +def test_install_normal_usage(driver, execute_in_page, complex_variant): + """ + Test of the normal package installation procedure with one mapping and, + depending on parameter, one or many resources. + """ + containers, assert_container_displayed = setup_view(driver, execute_in_page) + + assert execute_in_page('returnval(shw());') == [[], False] + + if complex_variant: + # The resource/mapping others depend on. + root_id = 'abcd-defg-ghij' + root_resource_id = f'resource_{root_id}' + root_mapping_id = f'mapping_{root_id}' + # Those ids are used to check the alphabetical ordering. + resource_ids = [f'resource_{letters}' for letters in ( + 'a', 'abcd', root_id, 'b', 'c', + 'd', 'defg', 'e', 'f', + 'g', 'ghij', 'h', 'i', 'j' + )] + files_count = 9 + else: + root_resource_id = f'resource_a' + root_mapping_id = f'mapping_a' + resource_ids = [root_resource_id] + files_count = 0 + + # Preview the installation of a resource, show resource's details, close + # the details and cancel installation. + execute_in_page('returnval(install_view.show(...arguments));', + 'https://hydril.la/', 'resource', root_resource_id) + + assert execute_in_page('returnval(shw());') == [['show'], True] + assert f'{root_resource_id}-2021.11.11-1'\ + in containers['install_preview'].text + assert_container_displayed('install_preview') + + entries = execute_in_page('returnval(ets().map(e => e.main_li.innerText));') + assert len(entries) == len(resource_ids) + # Verify alphabetical ordering. + assert all([id in text for id, text in zip(resource_ids, entries)]) + + assert not execute_in_page('returnval(ets()[0].old_ver);').is_displayed() + execute_in_page('returnval(ets()[0].details_but);').click() + assert 'resource_a' in containers['resource_preview_container'].text + assert_container_displayed('resource_preview_container') + + execute_in_page('returnval(install_view.resource_back_but);').click() + assert_container_displayed('install_preview') + + assert execute_in_page('returnval(shw());') == [['show'], True] + execute_in_page('returnval(install_view.cancel_but);').click() + assert execute_in_page('returnval(shw());') == [['show', 'hide'], False] + + # Preview the installation of a mapping and a resource, show mapping's + # details, close the details and commit the installation. + execute_in_page('returnval(install_view.show(...arguments));', + 'https://hydril.la/', 'mapping', + root_mapping_id, [2022, 5, 10]) + + assert execute_in_page('returnval(shw(2));') == [['show'], True] + assert_container_displayed('install_preview') + + entries = execute_in_page('returnval(ets().map(e => e.main_li.innerText));') + assert len(entries) == len(resource_ids) + 1 + assert f'{root_mapping_id}-2022.5.10' in entries[0] + # Verify alphabetical ordering. + assert all([id in text for id, text in zip(resource_ids, entries[1:])]) + + assert not execute_in_page('returnval(ets()[0].old_ver);').is_displayed() + execute_in_page('returnval(ets()[0].details_but);').click() + assert root_mapping_id in containers['mapping_preview_container'].text + assert_container_displayed('mapping_preview_container') + + execute_in_page('returnval(install_view.mapping_back_but);').click() + assert_container_displayed('install_preview') + + execute_in_page('returnval(install_view.install_but);').click() + installed = lambda d: 'ly installed!' in containers['dialog_container'].text + WebDriverWait(driver, 10).until(installed) + + assert execute_in_page('returnval(shw(2));') == [['show'], True] + execute_in_page('returnval(install_view.dialog_ctx.ok_but);').click() + assert execute_in_page('returnval(shw(2));') == [['show', 'hide'], False] + + # Verify the install + db_contents = get_db_contents(execute_in_page) + for item_type, ids in \ + [('mapping', {root_mapping_id}), ('resource', set(resource_ids))]: + assert set([it['identifier'] for it in db_contents[item_type]]) == ids + + assert all([len(db_contents[store]) == files_count + for store in ('file', 'file_uses')]) + + # Update the installed mapping to a newer version. + execute_in_page('returnval(install_view.show(...arguments));', + 'https://hydril.la/', 'mapping', root_mapping_id) + assert execute_in_page('returnval(shw(4));') == [['show'], True] + # resources are already in the newest versions, hence they should not appear + # in the install preview list. + assert execute_in_page('returnval(ets().length);') == 1 + # Mapping's version update information should be displayed. + assert execute_in_page('returnval(ets()[0].old_ver);').is_displayed() + execute_in_page('returnval(install_view.install_but);').click() + + WebDriverWait(driver, 10).until(installed) + + assert execute_in_page('returnval(shw(4));') == [['show'], True] + execute_in_page('returnval(install_view.dialog_ctx.ok_but);').click() + assert execute_in_page('returnval(shw(4));') == [['show', 'hide'], False] + + # Verify the newer version install. + old_db_contents, db_contents = db_contents, get_db_contents(execute_in_page) + old_db_contents['mapping'][0]['version'][-1] += 1 + assert db_contents['mapping'] == old_db_contents['mapping'] + + # All items are up to date - verify dialog is instead shown in this case. + execute_in_page('install_view.show(...arguments);', + 'https://hydril.la/', 'mapping', root_mapping_id) + + fetched = lambda d: 'Fetching ' not in containers['dialog_container'].text + WebDriverWait(driver, 10).until(fetched) + + assert 'Nothing to do - packages already installed.' \ + in containers['dialog_container'].text + assert_container_displayed('dialog_container') + + assert execute_in_page('returnval(shw(6));') == [['show'], True] + execute_in_page('returnval(install_view.dialog_ctx.ok_but);').click() + assert execute_in_page('returnval(shw(6));') == [['show', 'hide'], False] + +@pytest.mark.ext_data(install_ext_data) +@pytest.mark.usefixtures('webextension') +@pytest.mark.parametrize('message', [ + 'fetching_data', + 'failure_to_communicate_sendmessage', + 'HTTP_code_item', + 'invalid_JSON', + 'newer_API_version', + 'invalid_response_format', + 'indexeddb_error_item', + 'installing', + 'indexeddb_error_file_uses', + 'failure_to_communicate_fetch', + 'HTTP_code_file', + 'not_valid_text', + 'sha256_mismatch', + 'indexeddb_error_write' +]) +def test_install_dialogs(driver, execute_in_page, message): + """ + Test of various error and loading messages used in install view. + """ + containers, assert_container_displayed = setup_view(driver, execute_in_page) + + def dlg_buts(): + return execute_in_page( + '''{ + const dlg = install_view.dialog_ctx; + const ids = ['ask_buts', 'conf_buts']; + returnval(ids.filter(id => !dlg[id].classList.contains("hide"))); + }''') + + def dialog_txt(): + return execute_in_page( + 'returnval(install_view.dialog_ctx.msg.textContent);' + ) + + def assert_dlg(awaited_buttons, expected_msg, hides_install_view=True, + button_to_click='ok_but'): + WebDriverWait(driver, 10).until(lambda d: dlg_buts() == awaited_buttons) + + assert expected_msg == dialog_txt() + + execute_in_page( + f'returnval(install_view.dialog_ctx.{button_to_click});' + ).click() + + if hides_install_view: + assert execute_in_page('returnval(shw());') == \ + [['show', 'hide'], False] + + if message == 'fetching_data': + execute_in_page( + ''' + browser.tabs.sendMessage = () => new Promise(cb => {}); + install_view.show(...arguments); + ''', + 'https://hydril.la/', 'mapping', 'mapping_a') + + assert dlg_buts() == [] + assert dialog_txt() == 'Fetching data from repository...' + elif message == 'failure_to_communicate_sendmessage': + execute_in_page( + ''' + browser.tabs.sendMessage = () => Promise.resolve({error: "sth"}); + install_view.show(...arguments); + ''', + 'https://hydril.la/', 'mapping', 'mapping_a') + + assert_dlg(['conf_buts'], 'Failure to communicate with repository :(') + elif message == 'HTTP_code_item': + execute_in_page( + ''' + const response = {ok: false, status: 404}; + browser.tabs.sendMessage = () => Promise.resolve(response); + install_view.show(...arguments); + ''', + 'https://hydril.la/', 'mapping', 'mapping_a') + + assert_dlg(['conf_buts'], 'Repository sent HTTP code 404 :(') + elif message == 'invalid_JSON': + execute_in_page( + ''' + const response = {ok: true, status: 200, error_json: "sth"}; + browser.tabs.sendMessage = () => Promise.resolve(response); + install_view.show(...arguments); + ''', + 'https://hydril.la/', 'mapping', 'mapping_a') + + assert_dlg(['conf_buts'], "Repository's response is not valid JSON :(") + elif message == 'newer_API_version': + execute_in_page( + ''' + const response = { + ok: true, + status: 200, + json: {$schema: "https://hydrilla.koszko.org/schemas/api_mapping_description-2.1.schema.json"} + }; + browser.tabs.sendMessage = () => Promise.resolve(response); + install_view.show(...arguments); + ''', + 'https://hydril.la/', 'mapping', 'somemapping', [2, 1]) + + assert_dlg(['conf_buts'], + 'Mapping somemapping-2.1 was served using unsupported Hydrilla API version. You might need to update Haketilo.') + elif message == 'invalid_response_format': + execute_in_page( + ''' + const response = { + ok: true, + status: 200, + /* $schema is not a string as it should be. */ + json: {$schema: null} + }; + browser.tabs.sendMessage = () => Promise.resolve(response); + install_view.show(...arguments); + ''', + 'https://hydril.la/', 'resource', 'someresource') + + assert_dlg(['conf_buts'], + 'Resource someresource was served using a nonconforming response format.') + elif message == 'indexeddb_error_item': + execute_in_page( + ''' + haketilodb.idb_get = () => {throw "some error";}; + install_view.show(...arguments); + ''', + 'https://hydril.la/', 'mapping', 'mapping_a') + + assert_dlg(['conf_buts'], + "Error accessing Haketilo's internal database :(") + elif message == 'installing': + execute_in_page( + ''' + haketilodb.save_items = () => new Promise(() => {}); + returnval(install_view.show(...arguments)); + ''', + 'https://hydril.la/', 'mapping', 'mapping_b') + + execute_in_page('returnval(install_view.install_but);').click() + + assert dlg_buts() == [] + assert dialog_txt() == 'Installing...' + elif message == 'indexeddb_error_file_uses': + execute_in_page( + ''' + const old_idb_get = haketilodb.idb_get; + haketilodb.idb_get = function(transaction, store_name, identifier) { + if (store_name === "file_uses") + throw "some error"; + return old_idb_get(...arguments); + } + returnval(install_view.show(...arguments)); + ''', + 'https://hydril.la/', 'mapping', 'mapping_b') + + execute_in_page('returnval(install_view.install_but);').click() + + assert_dlg(['conf_buts'], + "Error accessing Haketilo's internal database :(") + elif message == 'failure_to_communicate_fetch': + execute_in_page( + ''' + fetch = () => {throw "some error";}; + returnval(install_view.show(...arguments)); + ''', + 'https://hydril.la/', 'mapping', 'mapping_b') + + execute_in_page('returnval(install_view.install_but);').click() + + assert_dlg(['conf_buts'], + 'Failure to communicate with repository :(') + elif message == 'HTTP_code_file': + execute_in_page( + ''' + fetch = () => Promise.resolve({ok: false, status: 400}); + returnval(install_view.show(...arguments)); + ''', + 'https://hydril.la/', 'mapping', 'mapping_b') + + execute_in_page('returnval(install_view.install_but);').click() + + assert_dlg(['conf_buts'], 'Repository sent HTTP code 400 :(') + elif message == 'not_valid_text': + execute_in_page( + ''' + const err = () => {throw "some error";}; + fetch = () => Promise.resolve({ok: true, status: 200, text: err}); + returnval(install_view.show(...arguments)); + ''', + 'https://hydril.la/', 'mapping', 'mapping_b') + + execute_in_page('returnval(install_view.install_but);').click() + + assert_dlg(['conf_buts'], "Repository's response is not valid text :(") + elif message == 'sha256_mismatch': + execute_in_page( + ''' + let old_fetch = fetch, url_used; + fetch = async function(url) { + url_used = url; + const response = await old_fetch(...arguments); + const text = () => response.text().then(t => t + ":d"); + return {ok: response.ok, status: response.status, text}; + } + returnval(install_view.show(...arguments)); + ''', + 'https://hydril.la/', 'mapping', 'mapping_b') + + execute_in_page('returnval(install_view.install_but);').click() + + get_url_used = lambda d: execute_in_page('returnval(url_used);') + url_used = WebDriverWait(driver, 10).until(get_url_used) + print ((url_used,)) + + assert dlg_buts() == ['conf_buts'] + assert dialog_txt() == \ + f'{url_used} served a file with different SHA256 cryptographic sum :(' + elif message == 'indexeddb_error_write': + execute_in_page( + ''' + haketilodb.save_items = () => {throw "some error";}; + returnval(install_view.show(...arguments)); + ''', + 'https://hydril.la/', 'mapping', 'mapping_b') + + execute_in_page('returnval(install_view.install_but);').click() + + assert_dlg(['conf_buts'], + "Error writing to Haketilo's internal database :(") + else: + raise Exception('made a typo in test function params?') diff --git a/test/haketilo_test/unit/test_item_list.py b/test/haketilo_test/unit/test_item_list.py new file mode 100644 index 0000000..35ed1d5 --- /dev/null +++ b/test/haketilo_test/unit/test_item_list.py @@ -0,0 +1,280 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - displaying list of resources/mappings +""" + +# This file is part of Haketilo +# +# Copyright (C) 2022, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +from selenium.webdriver.support.ui import WebDriverWait + +from ..extension_crafting import ExtraHTML +from ..script_loader import load_script +from .utils import * + +def make_sample_resource(identifier, long_name): + return { + 'source_name': 'hello', + 'source_copyright': [ + sample_file_ref('report.spdx'), + sample_file_ref('LICENSES/CC0-1.0.txt') + ], + 'type': 'resource', + 'identifier': identifier, + 'long_name': long_name, + 'uuid': 'a6754dcb-58d8-4b7a-a245-24fd7ad4cd68', + 'version': [2021, 11, 10], + 'revision': 1, + 'description': 'greets an apple', + 'dependencies': [{'identifier': 'hello-message'}], + 'scripts': [ + sample_file_ref('hello.js'), + sample_file_ref('bye.js') + ] + } + +def make_sample_mapping(identifier, long_name): + return { + 'source_name': 'example-org-fixes-new', + 'source_copyright': [ + sample_file_ref('report.spdx'), + sample_file_ref('LICENSES/CC0-1.0.txt') + ], + 'type': 'mapping', + 'identifier': identifier, + 'long_name': long_name, + 'uuid': '54d23bba-472e-42f5-9194-eaa24c0e3ee7', + 'version': [2022, 5, 10], + 'description': 'suckless something something', + 'payloads': { + 'https://example.org/a/*': { + 'identifier': 'some-KISS-resource' + }, + 'https://example.org/t/*': { + 'identifier': 'another-KISS-resource' + } + } + } + +def make_item(item_type, *args): + return make_sample_resource(*args) if item_type == 'resource' \ + else make_sample_mapping(*args) + +@pytest.mark.ext_data({ + 'background_script': broker_js, + 'extra_html': ExtraHTML('html/item_list.html', {}), + 'navigate_to': 'html/item_list.html' +}) +@pytest.mark.usefixtures('webextension') +@pytest.mark.parametrize('item_type', ['resource', 'mapping']) +def test_item_list_ordering(driver, execute_in_page, item_type): + """ + A test case of items list proper ordering. + """ + execute_in_page(load_script('html/item_list.js')) + + # Choose sample long names so as to test automatic sorting of items. + long_names = ['sample', 'sample it', 'Sample it', 'SAMPLE IT', + 'test', 'test it', 'Test it', 'TEST IT'] + # Let's operate on a reverse-sorted copy + long_names_reversed = [*long_names] + long_names_reversed.reverse() + + items = [make_item(item_type, f'it_{hex(2 * i + copy)[-1]}', name) + for i, name in enumerate(long_names_reversed) + for copy in (1, 0)] + # When adding/updating items this item will be updated at the end and this + # last update will be used to verify that a set of opertions completed. + extra_item = make_item(item_type, 'extraitem', 'extra item') + + # After this reversal items are sorted in the exact order they are expected + # to appear in the HTML list. + items.reverse() + + sample_data = { + 'resource': {}, + 'mapping': {}, + 'file': { + 'sha256': sample_files_by_sha256 + } + } + + indexes_added = set() + for iteration, to_include in enumerate([ + set([i for i in range(len(items)) if is_prime(i)]), + set([i for i in range(len(items)) + if not is_prime(i) and i & 1]), + set([i for i in range(len(items)) if i % 3 == 0]), + set([i for i in range(len(items)) + if i % 3 and not i & 1 and not is_prime(i)]), + set(range(len(items))) + ]): + # On the last iteration, re-add ALL items but with changed names. + if len(to_include) == len(items): + for it in items: + it['long_name'] = f'somewhat renamed {it["long_name"]}' + + items_to_inclue = [items[i] for i in sorted(to_include)] + sample_data[item_type] = sample_data_dict(items_to_inclue) + execute_in_page('returnval(haketilodb.save_items(arguments[0]));', + sample_data) + + extra_item['long_name'] = f'{iteration} {extra_item["long_name"]}' + sample_data[item_type] = sample_data_dict([extra_item]) + execute_in_page('returnval(haketilodb.save_items(arguments[0]));', + sample_data) + + if iteration == 0: + execute_in_page( + f''' + let list_ctx; + async function create_list() {{ + list_ctx = await {item_type}_list(); + document.body.append(list_ctx.main_div); + }} + returnval(create_list()); + ''') + + def lis_ready(driver): + return extra_item['long_name'] == execute_in_page( + 'returnval(list_ctx.ul.firstElementChild.textContent);' + ) + + indexes_added.update(to_include) + WebDriverWait(driver, 10).until(lis_ready) + + li_texts = execute_in_page( + ''' + var lis = [...list_ctx.ul.children].slice(1); + returnval(lis.map(li => li.textContent)); + ''') + assert li_texts == [items[i]['long_name'] for i in indexes_added] + + preview_texts = execute_in_page( + '''{ + const get_texts = + li => [li.click(), list_ctx.preview_container.textContent][1]; + returnval(lis.map(get_texts)); + }''') + + for i, text in zip(sorted(indexes_added), preview_texts): + assert items[i]['identifier'] in text + assert items[i]['long_name'] in text + +@pytest.mark.ext_data({ + 'background_script': broker_js, + 'extra_html': ExtraHTML('html/item_list.html', {}), + 'navigate_to': 'html/item_list.html' +}) +@pytest.mark.usefixtures('webextension') +@pytest.mark.parametrize('item_type', ['resource', 'mapping']) +def test_item_list_displaying(driver, execute_in_page, item_type): + """ + A test case of items list interaction with preview and dialog. + """ + execute_in_page(load_script('html/item_list.js')) + + items = [make_item(item_type, f'item{i}', f'Item {i}') for i in range(3)] + + sample_data = { + 'resource': {}, + 'mapping': {}, + 'file': { + 'sha256': sample_files_by_sha256 + } + } + sample_data[item_type] = sample_data_dict(items) + + preview_container, dialog_container, ul = execute_in_page( + f''' + let list_ctx, sample_data = arguments[0]; + async function create_list() {{ + await haketilodb.save_items(sample_data); + list_ctx = await {item_type}_list(); + document.body.append(list_ctx.main_div); + return [list_ctx.preview_container, list_ctx.dialog_container, + list_ctx.ul]; + }} + returnval(create_list()); + ''', + sample_data) + + assert not preview_container.is_displayed() + + # Check that preview is displayed correctly. + for i in range(3): + execute_in_page('list_ctx.ul.children[arguments[0]].click();', i) + assert preview_container.is_displayed() + text = preview_container.text + assert f'item{i}' in text + assert f'Item {i}' in text + + # Check that item removal confirmation dialog is displayed correctly. + execute_in_page('list_ctx.remove_but.click();') + WebDriverWait(driver, 10).until(lambda _: dialog_container.is_displayed()) + assert 'list_disabled' in ul.get_attribute('class') + assert not preview_container.is_displayed() + msg = execute_in_page('returnval(list_ctx.dialog_ctx.msg.textContent);') + assert msg == "Are you sure you want to delete 'item2'?" + + # Check that previewing other item is impossible while dialog is open. + execute_in_page('list_ctx.ul.children[0].click();') + assert dialog_container.is_displayed() + assert 'list_disabled' in ul.get_attribute('class') + assert not preview_container.is_displayed() + + # Check that queuing multiple removal confirmation dialogs is impossible. + execute_in_page('list_ctx.remove_but.click();') + + # Check that answering "No" causes the item not to be removed and unhides + # item preview. + execute_in_page('list_ctx.dialog_ctx.no_but.click();') + WebDriverWait(driver, 10).until(lambda _: preview_container.is_displayed()) + assert not dialog_container.is_displayed() + assert 'list_disabled' not in ul.get_attribute('class') + assert execute_in_page('returnval(list_ctx.ul.children.length);') == 3 + + # Check that item removal works properly. + def remove_current_item(): + execute_in_page('list_ctx.remove_but.click();') + WebDriverWait(driver, 10)\ + .until(lambda _: dialog_container.is_displayed()) + execute_in_page('list_ctx.dialog_ctx.yes_but.click();') + + remove_current_item() + + def item_deleted(driver): + return execute_in_page('returnval(list_ctx.ul.children.length);') == 2 + WebDriverWait(driver, 10).until(item_deleted) + assert not dialog_container.is_displayed() + assert not preview_container.is_displayed() + assert 'list_disabled' not in ul.get_attribute('class') + + execute_in_page('list_ctx.ul.children[1].click();') + + # Check that item removal failure causes the right error dialog to appear. + execute_in_page('haketilodb.finalize_transaction = () => {throw "sth";};') + remove_current_item() + WebDriverWait(driver, 10).until(lambda _: dialog_container.is_displayed()) + msg = execute_in_page('returnval(list_ctx.dialog_ctx.msg.textContent);') + assert msg == "Couldn't remove 'item1' :(" + + # Destroy item list. + assert True == execute_in_page( + ''' + const main_div = list_ctx.main_div; + destroy_list(list_ctx); + returnval(main_div.parentElement === null); + ''') diff --git a/test/haketilo_test/unit/test_item_preview.py b/test/haketilo_test/unit/test_item_preview.py new file mode 100644 index 0000000..fe9a98e --- /dev/null +++ b/test/haketilo_test/unit/test_item_preview.py @@ -0,0 +1,208 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - displaying resources and mappings details +""" + +# This file is part of Haketilo +# +# Copyright (C) 2022, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +from selenium.webdriver.support.ui import WebDriverWait +from selenium.common.exceptions import NoSuchWindowException + +from ..extension_crafting import ExtraHTML +from ..script_loader import load_script +from .utils import * + +@pytest.mark.ext_data({ + 'extra_html': ExtraHTML('html/item_preview.html', {}), + 'navigate_to': 'html/item_preview.html' +}) +@pytest.mark.usefixtures('webextension') +def test_resource_preview(driver, execute_in_page): + """ + A test case of the resource preview display function. + """ + execute_in_page(load_script('html/item_preview.js')) + + sample_resource = make_sample_resource() + + preview_div = execute_in_page( + ''' + let preview_object = resource_preview(arguments[0]); + document.body.append(preview_object.main_div); + returnval(preview_object.main_div); + ''', + sample_resource) + text = preview_div.text + + assert '...' not in text + + for string in [ + *filter(lambda v: type(v) is str, sample_resource.values()), + *[rr['identifier'] for rr in sample_resource['dependencies']], + *[c['file'] for k in ('source_copyright', 'scripts') + for c in sample_resource[k]], + item_version_string(sample_resource, True) + ]: + assert string in text + + sample_resource['identifier'] = 'hellopear' + sample_resource['long_name'] = 'Hello Pear' + sample_resource['description'] = 'greets a pear' + sample_resource['dependencies'] = [{'identifier': 'hello-msg'}] + for key in ('scripts', 'source_copyright'): + for file_ref in sample_resource[key]: + file_ref['file'] = file_ref['file'].replace('.', '_') + + preview_div = execute_in_page( + ''' + returnval(resource_preview(arguments[0], preview_object).main_div); + ''', + sample_resource) + text = preview_div.text + + for string in ['...', 'pple', 'hello-message', 'report.spdx', + 'LICENSES/CC0-1.0.txt', 'hello.js', 'bye.js']: + assert string not in text + + for string in ['hellopear', 'Hello Pear', 'hello-msg', 'greets a pear', + 'report_spdx', 'LICENSES/CC0-1_0_txt', 'hello_js', 'bye_js']: + assert string in text + +@pytest.mark.ext_data({ + 'extra_html': ExtraHTML('html/item_preview.html', {}), + 'navigate_to': 'html/item_preview.html' +}) +@pytest.mark.usefixtures('webextension') +def test_mapping_preview(driver, execute_in_page): + """ + A test case of the mapping preview display function. + """ + execute_in_page(load_script('html/item_preview.js')) + + sample_mapping = make_sample_mapping() + + preview_div = execute_in_page( + ''' + let preview_object = mapping_preview(arguments[0]); + document.body.append(preview_object.main_div); + returnval(preview_object.main_div); + ''', + sample_mapping) + text = preview_div.text + + assert '...' not in text + + for string in [ + *filter(lambda v: type(v) is str, sample_mapping.values()), + *[p['identifier'] for p in sample_mapping['payloads'].values()], + *[c['file'] for c in sample_mapping['source_copyright']], + item_version_string(sample_mapping) + ]: + assert string in text + + sample_mapping['identifier'] = 'example-org-bloated' + sample_mapping['long_name'] = 'Example.org Bloated', + sample_mapping['payloads'] = dict( + [(pat.replace('.org', '.com'), res_id) + for pat, res_id in sample_mapping['payloads'].items()] + ) + for file_ref in sample_mapping['source_copyright']: + file_ref['file'] = file_ref['file'].replace('.', '_') + + preview_div = execute_in_page( + ''' + returnval(mapping_preview(arguments[0], preview_object).main_div); + ''', + sample_mapping) + text = preview_div.text + + for string in ['...', 'inimal', 'example.org', 'report.spdx', + 'LICENSES/CC0-1.0.txt']: + assert string not in text + + for string in ['example-org-bloated', 'Example.org Bloated', 'example.com', + 'report_spdx', 'LICENSES/CC0-1_0_txt']: + assert string in text + +@pytest.mark.ext_data({ + 'background_script': broker_js, + 'extra_html': [ + ExtraHTML('html/item_preview.html', {}), + ExtraHTML('html/file_preview.html', {}, wrap_into_htmldoc=False) + ], + 'navigate_to': 'html/item_preview.html' +}) +@pytest.mark.usefixtures('webextension') +def test_file_preview_link(driver, execute_in_page): + """ + A test case of links created by preview functions that allow a + referenced file to be previewed. + """ + execute_in_page(load_script('html/item_preview.js')) + + sample_data = make_complete_sample_data() + sample_data['mapping'] = {} + execute_in_page('returnval(haketilodb.save_items(arguments[0]));', + sample_data) + + # Cause the "link" to `bye.js` to be invalid. + sample_resource = make_sample_resource() + sample_resource['scripts'][1]['sha256'] = 'dummy nonexistent hash' + + execute_in_page( + ''' + let resource_preview_object = resource_preview(arguments[0], undefined); + document.body.append(resource_preview_object.main_div); + ''', + sample_resource) + + window0 = driver.window_handles[0] + driver.find_element_by_link_text('hello.js').click() + + def blob_url_navigated(driver): + if len(driver.window_handles) < 2: + return + window1 = [wh for wh in driver.window_handles if wh != window0][0] + driver.switch_to.window(window1) + try: + return driver.current_url.startswith('blob') + except NoSuchWindowException: + pass + + WebDriverWait(driver, 10).until(blob_url_navigated) + + assert sample_files['hello.js']['contents'].strip() \ + in driver.find_element_by_tag_name("pre").text + + driver.close() + driver.switch_to.window(window0) + + driver.find_element_by_link_text('bye.js').click() + + def get_error_span(driver): + if len(driver.window_handles) < 2: + return + window1 = [wh for wh in driver.window_handles if wh != window0][0] + driver.switch_to.window(window1) + try: + return driver.find_element_by_id('error_msg') + except NoSuchWindowException: + pass + + error_span = WebDriverWait(driver, 10).until(get_error_span) + assert error_span.is_displayed() + assert "Couldn't find file in Haketilo's internal database :(" \ + in error_span.text diff --git a/test/haketilo_test/unit/test_patterns.py b/test/haketilo_test/unit/test_patterns.py new file mode 100644 index 0000000..f2eeaf8 --- /dev/null +++ b/test/haketilo_test/unit/test_patterns.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - URL patterns +""" + +# This file is part of Haketilo +# +# Copyright (C) 2021, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest + +from ..script_loader import load_script + +@pytest.mark.get_page('https://gotmyowndoma.in') +def test_regexes(execute_in_page): + """ + patterns.js contains regexes used for URL parsing. + Verify they work properly. + """ + execute_in_page(load_script('common/patterns.js')) + + valid_url = 'https://example.com/a/b?ver=1.2.3#heading2' + valid_url_rest = 'example.com/a/b?ver=1.2.3#heading2' + + # Test matching of URL protocol. + match = execute_in_page('returnval(proto_regex.exec(arguments[0]));', + valid_url) + assert match + assert match[1] == 'https' + assert match[2] == valid_url_rest + + match = execute_in_page('returnval(proto_regex.exec(arguments[0]));', + '://bad-url.missing/protocol') + assert match is None + + # Test matching of http(s) URLs. + match = execute_in_page('returnval(http_regex.exec(arguments[0]));', + valid_url_rest) + assert match + assert match[1] == 'example.com' + assert match[2] == '/a/b' + assert match[3] == '?ver=1.2.3' + + match = execute_in_page('returnval(http_regex.exec(arguments[0]));', + 'another.example.com') + assert match + assert match[1] == 'another.example.com' + assert match[2] == '' + assert match[3] == '' + + match = execute_in_page('returnval(http_regex.exec(arguments[0]));', + '/bad/http/example') + assert match == None + + # Test matching of file URLs. + match = execute_in_page('returnval(file_regex.exec(arguments[0]));', + '/good/file/example') + assert match + assert match[1] == '/good/file/example' + + # Test matching of ftp URLs. + match = execute_in_page('returnval(ftp_regex.exec(arguments[0]));', + 'example.com/a/b#heading2') + assert match + assert match[1] is None + assert match[2] == 'example.com' + assert match[3] == '/a/b' + + match = execute_in_page('returnval(ftp_regex.exec(arguments[0]));', + 'some_user@localhost') + assert match + assert match[1] == 'some_user@' + assert match[2] == 'localhost' + assert match[3] == '' + + match = execute_in_page('returnval(ftp_regex.exec(arguments[0]));', + '@bad.url/') + assert match is None + +@pytest.mark.get_page('https://gotmyowndoma.in') +def test_deconstruct_url(execute_in_page): + """ + patterns.js contains deconstruct_url() function that handles URL parsing. + Verify it works properly. + """ + execute_in_page(load_script('common/patterns.js')) + + deco = execute_in_page('returnval(deconstruct_url(arguments[0]));', + 'https://eXaMpLe.com/a/b?ver=1.2.3#heading2') + assert deco + assert deco['trailing_slash'] == False + assert deco['proto'] == 'https' + assert deco['domain'] == ['example', 'com'] + assert deco['path'] == ['a', 'b'] + + deco = execute_in_page('returnval(deconstruct_url(arguments[0]));', + 'http://**.example.com/') + assert deco + assert deco['trailing_slash'] == True + assert deco['proto'] == 'http' + assert deco['domain'] == ['**', 'example', 'com'] + assert deco['path'] == [] + + deco = execute_in_page('returnval(deconstruct_url(arguments[0]));', + 'ftp://user@ftp.example.com/all///passwords.txt/') + assert deco + assert deco['trailing_slash'] == True + assert deco['proto'] == 'ftp' + assert deco['domain'] == ['ftp', 'example', 'com'] + assert deco['path'] == ['all', 'passwords.txt'] + + deco = execute_in_page('returnval(deconstruct_url(arguments[0]));', + 'ftp://mirror.edu.pl.eu.org') + assert deco + assert deco['trailing_slash'] == False + assert deco['proto'] == 'ftp' + assert deco['domain'] == ['mirror', 'edu', 'pl', 'eu', 'org'] + assert deco['path'] == [] + + deco = execute_in_page('returnval(deconstruct_url(arguments[0]));', + 'file:///mnt/parabola_chroot///etc/passwd') + assert deco + assert deco['trailing_slash'] == False + assert deco['proto'] == 'file' + assert deco['path'] == ['mnt', 'parabola_chroot', 'etc', 'passwd'] + assert 'domain' not in deco + + for bad_url in [ + '://bad-url.missing/protocol', + 'http:/example.com/a/b', + 'unknown://example.com/a/b', + 'idontfancypineapple', + 'ftp://@example.org/', + 'https:///some/path/', + 'file://non-absolute/path' + ]: + with pytest.raises(Exception, match=r'Error in injected script'): + deco = execute_in_page('returnval(deconstruct_url(arguments[0]));', + bad_url) + + # at some point we might also consider testing url deconstruction with + # length limits... diff --git a/test/haketilo_test/unit/test_patterns_query_manager.py b/test/haketilo_test/unit/test_patterns_query_manager.py new file mode 100644 index 0000000..9fbc438 --- /dev/null +++ b/test/haketilo_test/unit/test_patterns_query_manager.py @@ -0,0 +1,307 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - building pattern tree and putting it in a content script +""" + +# This file is part of Haketilo +# +# Copyright (C) 2021,2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +import json +from selenium.webdriver.support.ui import WebDriverWait +from selenium.common.exceptions import TimeoutException + +from ..script_loader import load_script + +def simple_sample_mapping(patterns, fruit): + if type(patterns) is not list: + patterns = [patterns] + payloads = dict([(p, {'identifier': f'{fruit}-{p}'}) for p in patterns]) + return { + 'source_copyright': [], + 'type': 'mapping', + 'identifier': f'inject-{fruit}', + 'payloads': payloads + } + +def get_content_script_values(driver, content_script): + """ + Allow easy extraction of 'this.something = ...' values from generated + content script and verify the content script is syntactically correct. + """ + return driver.execute_script( + ''' + function value_holder() { + %s; + return this; + } + return value_holder.call({}); + ''' % content_script) + +# Fields that are not relevant for testing are omitted from these mapping +# definitions. +sample_mappings = [simple_sample_mapping(pats, fruit) for pats, fruit in [ + (['https://gotmyowndoma.in/index.html', + 'http://gotmyowndoma.in/index.html'], 'banana'), + (['https://***.gotmyowndoma.in/index.html', + 'https://**.gotmyowndoma.in/index.html', + 'https://*.gotmyowndoma.in/index.html', + 'https://gotmyowndoma.in/index.html'], 'orange'), + ('https://gotmyowndoma.in/index.html/***', 'grape'), + ('http://gotmyowndoma.in/index.html/***', 'melon'), + ('https://gotmyowndoma.in/index.html', 'peach'), + ('https://gotmyowndoma.in/*', 'pear'), + ('https://gotmyowndoma.in/**', 'raspberry'), + ('https://gotmyowndoma.in/***', 'strawberry'), + ('https://***.gotmyowndoma.in/index.html', 'apple'), + ('https://***.gotmyowndoma.in/*', 'avocado'), + ('https://***.gotmyowndoma.in/**', 'papaya'), + ('https://***.gotmyowndoma.in/***', 'kiwi') +]] + +sample_blocking = [f'http{s}://{dw}gotmyown%sdoma.in{i}{pw}' + for dw in ('', '***.', '**.', '*.') + for i in ('/index.html', '') + for pw in ('', '/', '/*') + for s in ('', 's')] +sample_blocking = [{'pattern': pattern % (i if i > 1 else ''), + 'allow': bool(i & 1)} + for i, pattern in enumerate(sample_blocking)] + +# Even though patterns_query_manager.js is normally meant to run from background +# page, some tests can be as well performed running it from a normal page. +@pytest.mark.get_page('https://gotmyowndoma.in') +def test_pqm_tree_building(driver, execute_in_page): + """ + patterns_query_manager.js tracks Haketilo's internal database and builds a + constantly-updated pattern tree based on its contents. Mock the database and + verify tree building works properly. + """ + execute_in_page(load_script('background/patterns_query_manager.js')) + # Mock IndexedDB and build patterns tree. + execute_in_page( + ''' + const [initial_mappings, initial_blocking] = arguments.slice(0, 2); + let mappingchange, blockingchange, settingchange; + + haketilodb.track.mapping = function (cb) { + mappingchange = cb; + + return [{}, initial_mappings]; + } + haketilodb.track.blocking = function (cb) { + blockingchange = cb; + + return [{}, initial_blocking]; + } + haketilodb.track.setting = function (cb) { + settingchange = cb; + + return [{}, [{name: "default_allow", value: true}]]; + } + + let last_script; + let unregister_called = 0; + async function register_mock(injection) + { + await new Promise(resolve => setTimeout(resolve, 1)); + last_script = injection.js[0].code; + return {unregister: () => unregister_called++}; + } + browser = {contentScripts: {register: register_mock}}; + + returnval(start("abracadabra")); + ''', + sample_mappings[0:2], sample_blocking[0:2]) + + found, tree, content_script, deregistrations = execute_in_page( + ''' + returnval([pqt.search(tree, arguments[0]).next().value, + tree, last_script, unregister_called]); + ''', + 'https://gotmyowndoma.in/index.html') + best_pattern = 'https://gotmyowndoma.in/index.html' + assert found == \ + dict([('~allow', 1), + *[(f'inject-{fruit}', {'identifier': f'{fruit}-{best_pattern}'}) + for fruit in ('banana', 'orange')]]) + cs_values = get_content_script_values(driver, content_script) + assert cs_values['haketilo_secret'] == 'abracadabra' + assert cs_values['haketilo_pattern_tree'] == tree + assert cs_values['haketilo_default_allow'] == True + assert deregistrations == 0 + + def condition_all_added(driver): + last_script = execute_in_page('returnval(last_script);') + cs_values = get_content_script_values(driver, last_script) + nums = [i for i in range(len(sample_blocking)) if i > 1] + return (cs_values['haketilo_default_allow'] == False and + all([('gotmyown%sdoma' % i) in last_script for i in nums]) and + all([m['identifier'] in last_script for m in sample_mappings])) + + execute_in_page( + '''{ + const new_setting_val = {name: "default_allow", value: false}; + settingchange({key: "default_allow", new_val: new_setting_val}); + for (const mapping of arguments[0]) + mappingchange({key: mapping.identifier, new_val: mapping}); + for (const blocking of arguments[1]) + blockingchange({key: blocking.pattern, new_val: blocking}); + }''', + sample_mappings[2:], sample_blocking[2:]) + WebDriverWait(driver, 10).until(condition_all_added) + + odd_mappings = \ + [m['identifier'] for i, m in enumerate(sample_mappings) if i & 1] + odd_blocking = \ + [b['pattern'] for i, b in enumerate(sample_blocking) if i & 1] + even_mappings = \ + [m['identifier'] for i, m in enumerate(sample_mappings) if 1 - i & 1] + even_blocking = \ + [b['pattern'] for i, b in enumerate(sample_blocking) if 1 - i & 1] + + def condition_odd_removed(driver): + last_script = execute_in_page('returnval(last_script);') + nums = [i for i in range(len(sample_blocking)) if i > 1 and 1 - i & 1] + return (all([id not in last_script for id in odd_mappings]) and + all([id in last_script for id in even_mappings]) and + all([p not in last_script for p in odd_blocking[1:]]) and + all([('gotmyown%sdoma' % i) in last_script for i in nums])) + + def condition_all_removed(driver): + content_script = execute_in_page('returnval(last_script);') + cs_values = get_content_script_values(driver, content_script) + return cs_values['haketilo_pattern_tree'] == {} + + execute_in_page( + ''' + arguments[0].forEach(identifier => mappingchange({key: identifier})); + arguments[1].forEach(pattern => blockingchange({key: pattern})); + ''', + odd_mappings, odd_blocking) + + WebDriverWait(driver, 10).until(condition_odd_removed) + + execute_in_page( + ''' + arguments[0].forEach(identifier => mappingchange({key: identifier})); + arguments[1].forEach(pattern => blockingchange({key: pattern})); + ''', + even_mappings, even_blocking) + + WebDriverWait(driver, 10).until(condition_all_removed) + + def condition_default_allowed_again(driver): + content_script = execute_in_page('returnval(last_script);') + cs_values = get_content_script_values(driver, content_script) + return cs_values['haketilo_default_allow'] == True + + execute_in_page( + '''{ + const new_setting_val = {name: "default_allow", value: true}; + settingchange({key: "default_allow", new_val: new_setting_val}); + }''') + + WebDriverWait(driver, 10).until(condition_default_allowed_again) + +content_js = ''' +let already_run = false; +this.haketilo_content_script_main = function() { + if (already_run) + return; + already_run = true; + document.documentElement.innerHTML = "
"; + document.getElementById("tree-json").innerText = + JSON.stringify(this.haketilo_pattern_tree); +} +if (this.haketilo_pattern_tree !== undefined) + this.haketilo_content_script_main(); +''' + +def background_js(): + pqm_js = load_script('background/patterns_query_manager.js', + "#IMPORT background/broadcast_broker.js") + return pqm_js + '; broadcast_broker.start(); start();' + +@pytest.mark.ext_data({ + 'content_script': content_js, + 'background_script': background_js +}) +@pytest.mark.usefixtures('webextension') +def test_pqm_script_injection(driver, execute_in_page): + # Let's open a normal page in a second window. Window 0 will be used to make + # changes to IndexedDB and window 1 to test the working of content scripts. + driver.execute_script('window.open("about:blank", "_blank");') + WebDriverWait(driver, 10).until(lambda d: len(d.window_handles) == 2) + windows = [*driver.window_handles] + + def get_tree_json(driver): + return driver.execute_script( + ''' + return (document.getElementById("tree-json") || {}).innerText; + ''') + + def run_content_script(): + driver.switch_to.window(windows[1]) + driver.get('https://gotmyowndoma.in/index.html') + windows[1] = driver.current_window_handle + try: + return WebDriverWait(driver, 10).until(get_tree_json) + except TimeoutException: + pass + + for attempt in range(2): + json_txt = run_content_script() + if json_txt and json.loads(json_txt) == {}: + break; + assert attempt != 1 + + driver.switch_to.window(windows[0]) + execute_in_page(load_script('common/indexeddb.js')) + + sample_data = { + 'mapping': dict([(sm['identifier'], {'1.0': sm}) + for sm in sample_mappings]), + 'resource': {}, + 'file': {} + } + execute_in_page('returnval(save_items(arguments[0]));', sample_data) + + for attempt in range(2): + tree_json = run_content_script() or '{}' + json.loads(tree_json) + if all([m['identifier'] in tree_json for m in sample_mappings]): + break + assert attempt != 1 + + driver.switch_to.window(windows[0]) + execute_in_page( + '''{ + const identifiers = arguments[0]; + async function remove_items() + { + const ctx = await start_items_transaction(["mapping"], {}); + for (const id of identifiers) + await remove_mapping(id, ctx); + await finalize_transaction(ctx); + } + returnval(remove_items()); + }''', + [sm['identifier'] for sm in sample_mappings]) + + for attempt in range(2): + json_txt = run_content_script() + if json_txt and json.loads(json_txt) == {}: + break; + assert attempt != 1 diff --git a/test/haketilo_test/unit/test_patterns_query_tree.py b/test/haketilo_test/unit/test_patterns_query_tree.py new file mode 100644 index 0000000..80bf554 --- /dev/null +++ b/test/haketilo_test/unit/test_patterns_query_tree.py @@ -0,0 +1,474 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - URL patterns +""" + +# This file is part of Haketilo +# +# Copyright (C) 2021, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest + +from ..script_loader import load_script + +@pytest.mark.get_page('https://gotmyowndoma.in') +def test_modify_branch(execute_in_page): + """ + patterns_query_tree.js contains Pattern Tree data structure that allows + arrays of string labels to be mapped to items. + Verify operations modifying a single branch of such tree work properly. + """ + execute_in_page(load_script('common/patterns_query_tree.js')) + execute_in_page( + ''' + let items_added; + let items_removed; + + function _item_adder(item, array) + { + items_added++; + return [...(array || []), item]; + } + + function item_adder(item) + { + items_added = 0; + return array => _item_adder(item, array); + } + + function _item_remover(array) + { + if (array !== null) { + items_removed++; + array.pop(); + } + return (array && array.length > 0) ? array : null; + } + + function item_remover() + { + items_removed = 0; + return _item_remover; + }''') + + # Let's construct some tree branch while checking that each addition gives + # the right result. + branch = execute_in_page( + '''{ + const branch = empty_node(); + modify_sequence(branch, ['com', 'example'], item_adder('some_item')); + returnval(branch); + }''') + assert branch == { + 'literal_match': None, + 'wildcard_matches': [None, None, None], + 'children': { + 'com': { + 'literal_match': None, + 'wildcard_matches': [None, None, None], + 'children': { + 'example': { + 'literal_match': ['some_item'], + 'wildcard_matches': [None, None, None], + 'children': { + } + } + } + } + } + } + + branch, items_added = execute_in_page( + '''{ + const branch = arguments[0]; + modify_sequence(branch, ['com', 'example'], item_adder('other_item')); + returnval([branch, items_added]); + }''', branch) + assert items_added == 1 + assert branch['children']['com']['children']['example']['literal_match'] \ + == ['some_item', 'other_item'] + + for i in range(3): + for expected_array in [['third_item'], ['third_item', '4th_item']]: + wildcard = '*' * (i + 1) + branch, items_added = execute_in_page( + '''{ + const branch = arguments[0]; + modify_sequence(branch, ['com', 'sample', arguments[1]], + item_adder(arguments[2])); + returnval([branch, items_added]); + }''', + branch, wildcard, expected_array[-1]) + assert items_added == 2 + sample = branch['children']['com']['children']['sample'] + assert sample['wildcard_matches'][i] == expected_array + assert sample['children'][wildcard]['literal_match'] \ + == expected_array + + branch, items_added = execute_in_page( + '''{ + const branch = arguments[0]; + modify_sequence(branch, ['org', 'koszko', '***', '123'], + item_adder('5th_item')); + returnval([branch, items_added]); + }''', + branch) + assert items_added == 1 + assert branch['children']['org']['children']['koszko']['children']['***']\ + ['children']['123']['literal_match'] == ['5th_item'] + + # Let's verify that removing a nonexistent element doesn't modify the tree. + branch2, items_removed = execute_in_page( + '''{ + const branch = arguments[0]; + modify_sequence(branch, ['com', 'not', 'registered', '*'], + item_remover()); + returnval([branch, items_removed]); + }''', + branch) + assert branch == branch2 + assert items_removed == 0 + + # Let's remove all elements in the tree branch while checking that each + # removal gives the right result. + branch, items_removed = execute_in_page( + '''{ + const branch = arguments[0]; + modify_sequence(branch, ['org', 'koszko', '***', '123'], + item_remover()); + returnval([branch, items_removed]); + }''', + branch) + assert items_removed == 1 + assert 'org' not in branch['children'] + + for i in range(3): + for expected_array in [['third_item'], None]: + wildcard = '*' * (i + 1) + branch, items_removed = execute_in_page( + '''{ + const branch = arguments[0]; + modify_sequence(branch, ['com', 'sample', arguments[1]], + item_remover()); + returnval([branch, items_removed]); + }''', + branch, wildcard) + assert items_removed == 2 + if i == 2 and expected_array == []: + break + sample = branch['children']['com']['children'].get('sample', {}) + assert sample.get('wildcard_matches', [None, None, None])[i] \ + == expected_array + assert sample.get('children', {}).get(wildcard, {})\ + .get('literal_match') == expected_array + + for i in range(2): + branch, items_removed = execute_in_page( + '''{ + const branch = arguments[0]; + modify_sequence(branch, ['com', 'example'], item_remover()); + returnval([branch, items_removed]); + }''', + branch) + assert items_removed == 1 + if i == 0: + assert branch['children']['com']['children']['example']\ + ['literal_match'] == ['some_item'] + else: + assert branch == { + 'literal_match': None, + 'wildcard_matches': [None, None, None], + 'children': { + } + } + +@pytest.mark.get_page('https://gotmyowndoma.in') +def test_search_branch(execute_in_page): + """ + patterns_query_tree.js contains Pattern Tree data structure that allows + arrays of string labels to be mapped to items. + Verify searching a single branch of such tree work properly. + """ + execute_in_page(load_script('common/patterns_query_tree.js')) + execute_in_page( + ''' + const item_adder = item => (array => [...(array || []), item]); + ''') + + # Let's construct some tree branch to test on. + execute_in_page( + ''' + var branch = empty_node(); + + for (const [item, sequence] of [ + ['(root)', []], + ['***', ['***']], + ['**', ['**']], + ['*', ['*']], + + ['a', ['a']], + ['A', ['a']], + ['b', ['b']], + + ['a/***', ['a', '***']], + ['A/***', ['a', '***']], + ['a/**', ['a', '**']], + ['A/**', ['a', '**']], + ['a/*', ['a', '*']], + ['A/*', ['a', '*']], + ['a/sth', ['a', 'sth']], + ['A/sth', ['a', 'sth']], + + ['b/***', ['b', '***']], + ['b/**', ['b', '**']], + ['b/*', ['b', '*']], + ['b/sth', ['b', 'sth']], + ]) + modify_sequence(branch, sequence, item_adder(item)); + ''') + + # Let's make the actual searches on our testing branch. + for sequence, expected in [ + ([], [{'(root)'}, {'***'}]), + (['a'], [{'a', 'A'}, {'a/***', 'A/***'}, {'*'}, {'***'}]), + (['b'], [{'b'}, {'b/***'}, {'*'}, {'***'}]), + (['c'], [ {'*'}, {'***'}]), + (['***'], [{'***'}, {'*'} ]), + (['**'], [{'**'}, {'*'}, {'***'}]), + (['**'], [{'**'}, {'*'}, {'***'}]), + (['*'], [{'*'}, {'***'}]), + + (['a', 'sth'], [{'a/sth', 'A/sth'}, {'a/*', 'A/*'}, {'a/***', 'A/***'}, {'**'}, {'***'}]), + (['b', 'sth'], [{'b/sth'}, {'b/*'}, {'b/***'}, {'**'}, {'***'}]), + (['a', 'hts'], [ {'a/*', 'A/*'}, {'a/***', 'A/***'}, {'**'}, {'***'}]), + (['b', 'hts'], [ {'b/*'}, {'b/***'}, {'**'}, {'***'}]), + (['a', '***'], [{'a/***', 'A/***'}, {'a/*', 'A/*'}, {'**'}, {'***'}]), + (['b', '***'], [{'b/***'}, {'b/*'}, {'**'}, {'***'}]), + (['a', '**'], [{'a/**', 'A/**'}, {'a/*', 'A/*'}, {'a/***', 'A/***'}, {'**'}, {'***'}]), + (['b', '**'], [{'b/**'}, {'b/*'}, {'b/***'}, {'**'}, {'***'}]), + (['a', '*'], [{'a/*', 'A/*'}, {'a/***', 'A/***'}, {'**'}, {'***'}]), + (['b', '*'], [{'b/*'}, {'b/***'}, {'**'}, {'***'}]), + + (['a', 'c', 'd'], [{'a/**', 'A/**'}, {'a/***', 'A/***'}, {'**'}, {'***'}]), + (['b', 'c', 'd'], [{'b/**'}, {'b/***'}, {'**'}, {'***'}]) + ]: + result = execute_in_page( + ''' + returnval([...search_sequence(branch, arguments[0])]); + ''', + sequence) + + try: + assert len(result) == len(expected) + + for expected_set, result_array in zip(expected, result): + assert len(expected_set) == len(result_array) + assert expected_set == set(result_array) + except Exception as e: + import sys + print('sequence:', sequence, '\nexpected:', expected, + '\nresult:', result, file=sys.stderr) + raise e from None + +@pytest.mark.get_page('https://gotmyowndoma.in') +def test_pattern_tree(execute_in_page): + """ + patterns_query_tree.js contains Pattern Tree data structure that allows + arrays of string labels to be mapped to items. + Verify operations on entire such tree work properly. + """ + execute_in_page(load_script('common/patterns_query_tree.js')) + + # Perform tests with all possible patterns for a simple URL. + url = 'https://example.com' + patterns = [ + 'https://example.com', + 'https://example.com/***', + 'https://***.example.com', + 'https://***.example.com/***' + ] + bad_patterns = [ + 'http://example.com', + 'https://a.example.com', + 'https://*.example.com', + 'https://**.example.com', + 'https://example.com/a', + 'https://example.com/*', + 'https://example.com/**', + ] + + expected = [{'key': p} for p in patterns] + + tree, result = execute_in_page( + '''{ + const tree = pattern_tree_make(); + for (const pattern of arguments[0].concat(arguments[1])) { + pattern_tree_register(tree, pattern, 'key', pattern); + pattern_tree_register(tree, pattern + '/', 'key', pattern + '/'); + } + returnval([tree, [...pattern_tree_search(tree, arguments[2])]]); + }''', + patterns, bad_patterns, url) + assert expected == result + + # Also verify that deregistering half of the good patterns works correctly. + patterns_removed = [pattern for i, pattern in enumerate(patterns) if i % 2] + patterns = [pattern for i, pattern in enumerate(patterns) if not (i % 2)] + expected = [{'key': p} for p in patterns] + tree, result = execute_in_page( + '''{ + const tree = arguments[0]; + for (const pattern of arguments[1]) { + pattern_tree_deregister(tree, pattern, 'key'); + pattern_tree_deregister(tree, pattern + '/', 'key'); + } + returnval([tree, [...pattern_tree_search(tree, arguments[2])]]); + }''', + tree, patterns_removed, url) + assert expected == result + + # Also verify that deregistering all the patterns works correctly. + tree = execute_in_page( + '''{ + const tree = arguments[0]; + for (const pattern of arguments[1].concat(arguments[2])) { + pattern_tree_deregister(tree, pattern, 'key'); + pattern_tree_deregister(tree, pattern + '/', 'key'); + } + returnval(tree); + }''', + tree, patterns, bad_patterns) + assert tree == {} + + # Perform tests with all possible patterns for a complex URL. + url = 'http://settings.query.example.com/google/tries/destroy/adblockers//' + patterns = [ + 'http://settings.query.example.com/google/tries/destroy/adblockers', + 'http://settings.query.example.com/google/tries/destroy/adblockers/***', + 'http://settings.query.example.com/google/tries/destroy/*', + 'http://settings.query.example.com/google/tries/destroy/***', + 'http://settings.query.example.com/google/tries/**', + 'http://settings.query.example.com/google/tries/***', + 'http://settings.query.example.com/google/**', + 'http://settings.query.example.com/google/***', + 'http://settings.query.example.com/**', + 'http://settings.query.example.com/***', + + 'http://***.settings.query.example.com/google/tries/destroy/adblockers', + 'http://***.settings.query.example.com/google/tries/destroy/adblockers/***', + 'http://***.settings.query.example.com/google/tries/destroy/*', + 'http://***.settings.query.example.com/google/tries/destroy/***', + 'http://***.settings.query.example.com/google/tries/**', + 'http://***.settings.query.example.com/google/tries/***', + 'http://***.settings.query.example.com/google/**', + 'http://***.settings.query.example.com/google/***', + 'http://***.settings.query.example.com/**', + 'http://***.settings.query.example.com/***', + 'http://*.query.example.com/google/tries/destroy/adblockers', + 'http://*.query.example.com/google/tries/destroy/adblockers/***', + 'http://*.query.example.com/google/tries/destroy/*', + 'http://*.query.example.com/google/tries/destroy/***', + 'http://*.query.example.com/google/tries/**', + 'http://*.query.example.com/google/tries/***', + 'http://*.query.example.com/google/**', + 'http://*.query.example.com/google/***', + 'http://*.query.example.com/**', + 'http://*.query.example.com/***', + 'http://***.query.example.com/google/tries/destroy/adblockers', + 'http://***.query.example.com/google/tries/destroy/adblockers/***', + 'http://***.query.example.com/google/tries/destroy/*', + 'http://***.query.example.com/google/tries/destroy/***', + 'http://***.query.example.com/google/tries/**', + 'http://***.query.example.com/google/tries/***', + 'http://***.query.example.com/google/**', + 'http://***.query.example.com/google/***', + 'http://***.query.example.com/**', + 'http://***.query.example.com/***', + 'http://**.example.com/google/tries/destroy/adblockers', + 'http://**.example.com/google/tries/destroy/adblockers/***', + 'http://**.example.com/google/tries/destroy/*', + 'http://**.example.com/google/tries/destroy/***', + 'http://**.example.com/google/tries/**', + 'http://**.example.com/google/tries/***', + 'http://**.example.com/google/**', + 'http://**.example.com/google/***', + 'http://**.example.com/**', + 'http://**.example.com/***', + 'http://***.example.com/google/tries/destroy/adblockers', + 'http://***.example.com/google/tries/destroy/adblockers/***', + 'http://***.example.com/google/tries/destroy/*', + 'http://***.example.com/google/tries/destroy/***', + 'http://***.example.com/google/tries/**', + 'http://***.example.com/google/tries/***', + 'http://***.example.com/google/**', + 'http://***.example.com/google/***', + 'http://***.example.com/**', + 'http://***.example.com/***' + ] + bad_patterns = [ + 'https://settings.query.example.com/google/tries/destroy/adblockers', + 'http://settings.query.example.com/google/tries/destroy/adblockers/a', + 'http://settings.query.example.com/google/tries/destroy/adblockers/*', + 'http://settings.query.example.com/google/tries/destroy/adblockers/**', + 'http://settings.query.example.com/google/tries/destroy/a', + 'http://settings.query.example.com/google/tries/destroy/**', + 'http://settings.query.example.com/google/tries/*', + 'http://a.settings.query.example.com/google/tries/destroy/adblockers', + 'http://*.settings.query.example.com/google/tries/destroy/adblockers', + 'http://**.settings.query.example.com/google/tries/destroy/adblockers', + 'http://a.query.example.com/google/tries/destroy/adblockers', + 'http://**.query.example.com/google/tries/destroy/adblockers', + 'http://*.example.com/google/tries/destroy/adblockers' + ] + + expected = [{'key': p + s} for p in patterns for s in ['/', '']] + + tree, result = execute_in_page( + '''{ + const tree = pattern_tree_make(); + for (const pattern of arguments[0].concat(arguments[1])) { + pattern_tree_register(tree, pattern, 'key', pattern); + pattern_tree_register(tree, pattern + '/', 'key', pattern + '/'); + } + returnval([tree, [...pattern_tree_search(tree, arguments[2])]]); + }''', + patterns, bad_patterns, url) + assert expected == result + + # Also verify that deregistering all patterns with trailing slash works + # correctly. + expected = [{'key': p} for p in patterns] + tree, result = execute_in_page( + '''{ + const tree = arguments[0]; + for (const pattern of arguments[1]) + pattern_tree_deregister(tree, pattern + '/', 'key'); + returnval([tree, [...pattern_tree_search(tree, arguments[2])]]); + }''', + tree, patterns, url) + assert expected == result + + # Also verify that deregistering all the patterns works correctly. + tree = execute_in_page( + '''{ + const tree = arguments[0]; + for (const pattern of arguments[1]) + pattern_tree_deregister(tree, pattern, 'key'); + for (const pattern of arguments[2]) { + pattern_tree_deregister(tree, pattern, 'key'); + pattern_tree_deregister(tree, pattern + '/', 'key'); + } + returnval(tree); + }''', + tree, patterns, bad_patterns) + assert tree == {} diff --git a/test/haketilo_test/unit/test_payload_create.py b/test/haketilo_test/unit/test_payload_create.py new file mode 100644 index 0000000..9689c37 --- /dev/null +++ b/test/haketilo_test/unit/test_payload_create.py @@ -0,0 +1,248 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - using a form to create simple site payload +""" + +# This file is part of Haketilo +# +# Copyright (C) 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +import re +from hashlib import sha256 + +from selenium.webdriver.support.ui import WebDriverWait + +from ..extension_crafting import ExtraHTML +from ..script_loader import load_script +from .utils import * + +uuidv4_re = re.compile( + r'^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$', + re.IGNORECASE +) + +sample_patterns = ''' +http://example.com/*** + +https://*.example.org/**''' + +sample_form_data = { + 'identifier': 'someid', + 'long_name': 'Some Name', + 'description': 'blah blah blah', + 'patterns': sample_patterns, + 'script': sample_files['hello.js']['contents'] +} + +def fill_form_with_sample_data(execute_in_page, sample_data_override={}, + form_ctx='form_ctx'): + form_data = sample_form_data.copy() + form_data.update(sample_data_override) + execute_in_page( + f''' + for (const [key, value] of Object.entries(arguments[0])) + {form_ctx}[key].value = value; + ''', + form_data) + return form_data + +cleared_form_inputs = { + 'identifier': '', + 'long_name': '', + 'description': '', + 'patterns': 'https://example.com/***', + 'script': 'console.log("Hello, World!");' +} +def assert_form_contents(execute_in_page, inputs=cleared_form_inputs): + inputs_keys = [*inputs.keys()] + values = execute_in_page( + 'returnval(arguments[0].map(i => form_ctx[i].value));', + inputs_keys + ) + for key, value in zip(inputs_keys, values): + assert inputs[key] == value + +@pytest.mark.ext_data({ + 'background_script': broker_js, + 'extra_html': ExtraHTML('html/payload_create.html', {}), + 'navigate_to': 'html/payload_create.html' +}) +@pytest.mark.usefixtures('webextension') +def test_payload_create_normal_usage(driver, execute_in_page): + """ + A test case of normal usage of simple payload creation form. + """ + execute_in_page(load_script('html/payload_create.js')) + + create_but, form_container, dialog_container = execute_in_page( + ''' + const form_ctx = payload_create_form(); + document.body.append(form_ctx.main_div); + returnval([form_ctx.create_but, form_ctx.form_container, + form_ctx.dialog_container]); + ''') + + assert patterns_doc_url == \ + driver.find_element_by_link_text('URL patterns').get_attribute('href') + + assert form_container.is_displayed() + assert not dialog_container.is_displayed() + + assert_form_contents(execute_in_page) + + form_data = fill_form_with_sample_data(execute_in_page) + + create_but.click() + + assert not form_container.is_displayed() + assert dialog_container.is_displayed() + + def success_reported(driver): + return 'Successfully saved payload' in dialog_container.text + + WebDriverWait(driver, 10).until(success_reported) + execute_in_page('form_ctx.dialog_ctx.ok_but.click();') + + assert form_container.is_displayed() + assert not dialog_container.is_displayed() + + def assert_db_contents(): + db_contents = get_db_contents(execute_in_page) + + assert uuidv4_re.match(db_contents['resource'][0]['uuid']) + + localid = f'local-{form_data["identifier"]}' + long_name = form_data['long_name'] or form_data['identifier'] + payloads = dict([(pat, {'identifier': localid}) + for pat in form_data['patterns'].split('\n') if pat]) + + assert db_contents['resource'] == [{ + 'source_name': localid, + 'source_copyright': [], + 'type': 'resource', + 'identifier': localid, + 'uuid': db_contents['resource'][0]['uuid'], + 'version': [1], + 'description': form_data['description'], + 'dependencies': [], + 'long_name': long_name, + 'scripts': [{ + 'file': 'payload.js', + 'sha256': sha256(form_data['script'].encode()).digest().hex() + }] + }] + + assert uuidv4_re.match(db_contents['mapping'][0]['uuid']) + assert db_contents['mapping'] == [{ + 'source_name': localid, + 'source_copyright': [], + 'type': 'mapping', + 'identifier': localid, + 'uuid': db_contents['mapping'][0]['uuid'], + 'version': [1], + 'description': form_data['description'], + 'long_name': long_name, + 'payloads': payloads + }] + + assert_db_contents() + + form_data = fill_form_with_sample_data(execute_in_page, { + 'long_name': '', + 'description': 'bam bam bam', + 'patterns': 'https://new.example.com/***', + 'script': sample_files['bye.js']['contents'] + }) + + create_but.click() + + for type in ('Resource', 'Mapping'): + def override_asked(driver): + return f"{type} 'local-someid' already exists. Override?" \ + in dialog_container.text + WebDriverWait(driver, 10).until(override_asked) + execute_in_page('form_ctx.dialog_ctx.yes_but.click();') + + assert_db_contents() + +@pytest.mark.ext_data({ + 'background_script': broker_js, + 'extra_html': ExtraHTML('html/payload_create.html', {}), + 'navigate_to': 'html/payload_create.html' +}) +@pytest.mark.usefixtures('webextension') +def test_payload_create_errors(driver, execute_in_page): + """ + A test case of various error the simple payload form might show. + """ + execute_in_page(load_script('html/payload_create.js')) + + create_but, dialog_container = execute_in_page( + ''' + const form_ctx = payload_create_form(); + document.body.append(form_ctx.main_div); + returnval([form_ctx.create_but, form_ctx.dialog_container]); + ''') + + for data_override, expected_msg in [ + ({'identifier': ''}, "The 'identifier' field is required!"), + ({'identifier': ':('}, 'Identifier may only contain '), + ({'script': ''}, "The 'script' field is required!"), + ({'patterns': ''}, "The 'URL patterns' field is required!"), + ({'patterns': ':d'}, "':d' is not a valid URL pattern. See here for more details."), + ({'patterns': '\n'.join(['http://example.com'] * 2)}, + "Pattern 'http://example.com' specified multiple times!") + ]: + # Attempt creating the payload + form_data = fill_form_with_sample_data(execute_in_page, data_override) + create_but.click() + # Verify the error message + assert expected_msg in dialog_container.text + + # Verify patterns documentation link. + if expected_msg == {'patterns': ':d'}: + doc_link_elem = driver.find_element_by_link_text('here') + assert doc_link.get_attribute('href') == patterns_doc_url + + # Verify the form was NOT cleared upon failed saving. + execute_in_page('form_ctx.dialog_ctx.ok_but.click();') + assert_form_contents(execute_in_page, form_data) + + # Add a sample item and attempt overriding it. + fill_form_with_sample_data(execute_in_page) + create_but.click() + WebDriverWait(driver, 10).until(lambda _: 'Succes' in dialog_container.text) + execute_in_page('form_ctx.dialog_ctx.ok_but.click();') + + # Verify that denying override leads to saving failure. + form_data = fill_form_with_sample_data(execute_in_page) + create_but.click() + WebDriverWait(driver, 10).until(lambda _: 'Overri' in dialog_container.text) + execute_in_page('form_ctx.dialog_ctx.no_but.click();') + assert 'Failed to save payload :(' in dialog_container.text + execute_in_page('form_ctx.dialog_ctx.ok_but.click();') + assert_form_contents(execute_in_page, form_data) + + # Verify that IndexedDB errors get caught and reported as saving failures. + execute_in_page('haketilodb.get = async () => {throw "someerror";}') + form_data = fill_form_with_sample_data(execute_in_page, {'identifier': 'o'}) + create_but.click() + WebDriverWait(driver, 10).until(lambda _: 'Failed' in dialog_container.text) + execute_in_page('form_ctx.dialog_ctx.ok_but.click();') + assert_form_contents(execute_in_page, form_data) + + # Verify that the loading message gets shown during IndexedDB operations. + execute_in_page('haketilodb.get = () => new Promise(cb => null);') + create_but.click() + assert 'Saving payload...' in dialog_container.text diff --git a/test/haketilo_test/unit/test_policy_deciding.py b/test/haketilo_test/unit/test_policy_deciding.py new file mode 100644 index 0000000..75b35ac --- /dev/null +++ b/test/haketilo_test/unit/test_policy_deciding.py @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - determining what to do on a given web page +""" + +# This file is part of Haketilo +# +# Copyright (C) 2021, Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import re +from hashlib import sha256 +import pytest + +from ..script_loader import load_script + +csp_re = re.compile(r'^\S+\s+\S+;(?:\s+\S+\s+\S+;)*$') +rule_re = re.compile(r'^\s*(?P\S+)\s+(?P\S+)$') +def parse_csp(csp): + ''' + Parsing of CSP string into a dict. A simplified format of CSP is assumed. + ''' + assert csp_re.match(csp) + + result = {} + + for rule in csp.split(';')[:-1]: + match = rule_re.match(rule) + result[match.group('src_kind')] = match.group('allowed_origins') + + return result + +@pytest.mark.get_page('https://gotmyowndoma.in') +def test_decide_policy(execute_in_page): + """ + policy.js contains code that, using a Pattern Query Tree instance and a URL, + decides what Haketilo should do on a page opened at that URL, i.e. whether + it should block or allow script execution and whether it should inject its + own scripts and which ones. Test that the policy object gets constructed + properly. + """ + execute_in_page(load_script('common/policy.js')) + + policy = execute_in_page( + ''' + returnval(decide_policy(pqt.make(), "http://unkno.wn/", true, "abcd")); + ''') + assert policy['allow'] == True + for prop in ('mapping', 'payload', 'nonce', 'csp', 'error'): + assert prop not in policy + + policy = execute_in_page( + '''{ + const tree = pqt.make(); + pqt.register(tree, "http://kno.wn", "~allow", 1); + returnval(decide_policy(tree, "http://kno.wn/", false, "abcd")); + }''') + assert policy['allow'] == True + assert policy['mapping'] == '~allow' + for prop in ('payload', 'nonce', 'csp', 'error'): + assert prop not in policy + + policy = execute_in_page( + ''' + returnval(decide_policy(pqt.make(), "http://unkno.wn/", false, "abcd")); + ''' + ) + assert policy['allow'] == False + for prop in ('mapping', 'payload', 'nonce', 'error'): + assert prop not in policy + assert parse_csp(policy['csp']) == { + 'prefetch-src': "'none'", + 'script-src-attr': "'none'", + 'script-src': "'none'", + 'script-src-elem': "'none'" + } + + policy = execute_in_page( + '''{ + const tree = pqt.make(); + pqt.register(tree, "http://kno.wn", "~allow", 0); + returnval(decide_policy(tree, "http://kno.wn/", true, "abcd")); + }''') + assert policy['allow'] == False + assert policy['mapping'] == '~allow' + for prop in ('payload', 'nonce', 'error'): + assert prop not in policy + assert parse_csp(policy['csp']) == { + 'prefetch-src': "'none'", + 'script-src-attr': "'none'", + 'script-src': "'none'", + 'script-src-elem': "'none'" + } + + policy = execute_in_page( + '''{ + const tree = pqt.make(); + pqt.register(tree, "http://kno.wn", "m1", {identifier: "res1"}); + returnval(decide_policy(tree, "http://kno.wn/", true, "abcd")); + }''') + assert policy['allow'] == False + assert policy['mapping'] == 'm1' + assert policy['payload'] == {'identifier': 'res1'} + assert 'error' not in policy + assert policy['nonce'] == \ + sha256('m1:res1:http://kno.wn/:abcd'.encode()).digest().hex() + assert parse_csp(policy['csp']) == { + 'prefetch-src': f"'none'", + 'script-src-attr': f"'none'", + 'script-src': f"'nonce-{policy['nonce']}'", + 'script-src-elem': f"'nonce-{policy['nonce']}'" + } + + policy = execute_in_page( + 'returnval(decide_policy(pqt.make(), "", true, "abcd"));' + ) + assert policy['allow'] == False + assert policy['error'] == {'haketilo_error_type': 'deciding_policy'} + for prop in ('mapping', 'payload', 'nonce'): + assert prop not in policy + assert parse_csp(policy['csp']) == { + 'prefetch-src': "'none'", + 'script-src-attr': "'none'", + 'script-src': "'none'", + 'script-src-elem': "'none'" + } diff --git a/test/haketilo_test/unit/test_policy_enforcing.py b/test/haketilo_test/unit/test_policy_enforcing.py new file mode 100644 index 0000000..4b7c173 --- /dev/null +++ b/test/haketilo_test/unit/test_policy_enforcing.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - enforcing script blocking policy from content script +""" + +# This file is part of Haketilo +# +# Copyright (C) 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +import json +import urllib.parse +from selenium.webdriver.support.ui import WebDriverWait + +from ..script_loader import load_script +from .utils import are_scripts_allowed + +# For simplicity, we'll use one nonce in all test cases. +nonce = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' + +allow_policy = {'allow': True} +block_policy = { + 'allow': False, + 'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'none'; script-src-elem 'none'; frame-src http://* https://*;" +} +payload_policy = { + 'mapping': 'somemapping', + 'payload': {'identifier': 'someresource'}, + 'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'nonce-{nonce}'; script-src-elem 'nonce-{nonce}';" +} + +content_script = load_script('content/policy_enforcing.js') + ''';{ +const smuggled_what_to_do = /^[^#]*#?(.*)$/.exec(document.URL)[1]; +const what_to_do = smuggled_what_to_do === "" ? {policy: {allow: true}} : + JSON.parse(decodeURIComponent(smuggled_what_to_do)); + +if (what_to_do.csp_off) { + const orig_DOMParser = window.DOMParser; + window.DOMParser = function() { + const parser = new orig_DOMParser(); + this.parseFromString = () => parser.parseFromString('', 'text/html'); + } +} + +enforce_blocking(what_to_do.policy); +}''' + +def get(driver, page, what_to_do): + driver.get(page + '#' + urllib.parse.quote(json.dumps(what_to_do))) + driver.execute_script('window.before_reload = true; location.reload();') + done = lambda _: not driver.execute_script('return window.before_reload;') + WebDriverWait(driver, 10).until(done) + +@pytest.mark.ext_data({'content_script': content_script}) +@pytest.mark.usefixtures('webextension') +# Under Mozilla we use several mechanisms of script blocking. Some serve as +# fallbacks in case others break. CSP one of those mechanisms. Here we run the +# test once with CSP blocking on and once without it. This allows us to verify +# that the CSP-less blocking approaches by themselves also work. We don't do the +# reverse (CSP on and other mechanisms off) because CSP rules added through +# injection are not reliable enough - they do not always take effect +# immediately and there's nothing we can do to fix it. +@pytest.mark.parametrize('csp_off_setting', [{}, {'csp_off': True}]) +def test_policy_enforcing_html(driver, execute_in_page, csp_off_setting): + """ + A test case of sanitizing -

d0 (channel `somebodyoncetoldme`)

-
-

d1 (channel `worldisgonnarollme`)

-
-

d2 (both channels)

-
-''' - -@pytest.mark.ext_data({ - 'background_script': broker_js, - 'test_page': test_page_html, - 'extra_files': { - 'testpage.js': lambda: load_script('common/broadcast.js') - } -}) -@pytest.mark.usefixtures('webextension') -def test_broadcast(driver, execute_in_page, wait_elem_text): - """ - A test that verifies the broadcasting system based on WebExtension messaging - API and implemented in `background/broadcast_broker.js` and - `common/broadcast.js` works correctly. - """ - # The broadcast facility is meant to enable message distribution between - # multiple contexts (e.g. different tabs/windows). Let's open the same - # extension's test page in a second window. - driver.execute_script( - ''' - window.open(window.location.href, "_blank"); - window.open(window.location.href, "_blank"); - ''') - WebDriverWait(driver, 10).until(lambda d: len(d.window_handles) == 3) - windows = [*driver.window_handles] - - # Let's first test if a simple message can be successfully broadcasted - driver.switch_to.window(windows[0]) - execute_in_page( - ''' - const divs = [0, 1, 2].map(n => document.getElementById("d" + n)); - let appender = n => (t => divs[n].append("\\n" + `[${t[0]}, ${t[1]}]`)); - let listener0 = listener_connection(appender(0)); - subscribe(listener0, "somebodyoncetoldme"); - ''') - - driver.switch_to.window(windows[1]) - execute_in_page( - ''' - let sender0 = sender_connection(); - out(sender0, "somebodyoncetoldme", "iaintthesharpesttool"); - ''') - - driver.switch_to.window(windows[0]) - wait_elem_text('d0', '[somebodyoncetoldme, iaintthesharpesttool]') - - # Let's add 2 more listeners - driver.switch_to.window(windows[0]) - execute_in_page( - ''' - let listener1 = listener_connection(appender(1)); - subscribe(listener1, "worldisgonnarollme"); - let listener2 = listener_connection(appender(2)); - subscribe(listener2, "worldisgonnarollme"); - subscribe(listener2, "somebodyoncetoldme"); - ''') - - # Let's send one message to one channel and one to the other. Verify they - # were received by the rght listeners. - driver.switch_to.window(windows[1]) - execute_in_page( - ''' - out(sender0, "somebodyoncetoldme", "intheshed"); - out(sender0, "worldisgonnarollme", "shewaslooking"); - ''') - - driver.switch_to.window(windows[0]) - wait_elem_text('d0', 'intheshed') - wait_elem_text('d1', 'shewaslooking') - wait_elem_text('d2', 'intheshed') - wait_elem_text('d2', 'shewaslooking') - - text = execute_in_page('returnval(divs[0].innerText);') - assert 'shewaslooking' not in text - text = execute_in_page('returnval(divs[1].innerText);') - assert 'intheshed' not in text - - # Let's create a second sender in third window and use it to send messages - # with the 'prepare' feature. - driver.switch_to.window(windows[2]) - execute_in_page( - ''' - let sender1 = sender_connection(); - prepare(sender1, "somebodyoncetoldme", "kindadumb"); - out(sender1, "worldisgonnarollme", "withherfinger"); - ''') - - driver.switch_to.window(windows[0]) - wait_elem_text('d1', 'withherfinger') - text = execute_in_page('returnval(divs[0].innerText);') - assert 'kindadumb' not in text - - driver.switch_to.window(windows[2]) - execute_in_page('flush(sender1);') - - driver.switch_to.window(windows[0]) - wait_elem_text('d0', 'kindadumb') - - # Let's verify that prepare()'d messages are properly discarded when - # discard() is called. - driver.switch_to.window(windows[2]) - execute_in_page( - ''' - prepare(sender1, "somebodyoncetoldme", "andherthumb"); - discard(sender1); - prepare(sender1, "somebodyoncetoldme", "andhermiddlefinger"); - flush(sender1); - ''') - - driver.switch_to.window(windows[0]) - wait_elem_text('d0', 'andhermiddlefinger') - text = execute_in_page('returnval(divs[0].innerText);') - assert 'andherthumb' not in text - - # Let's verify prepare()'d messages are properly auto-flushed when the other - # end of the connection gets killed (e.g. because browser tab gets closed). - driver.switch_to.window(windows[2]) - execute_in_page( - ''' - prepare(sender1, "worldisgonnarollme", "intheshape", 500); - ''') - driver.close() - - driver.switch_to.window(windows[0]) - wait_elem_text('d2', 'intheshape') - - # Verify listener's connection gets closed properly. - execute_in_page('close(listener0); close(listener1);') - - driver.switch_to.window(windows[1]) - execute_in_page('out(sender0, "worldisgonnarollme", "ofanL");') - execute_in_page('out(sender0, "somebodyoncetoldme", "forehead");') - - driver.switch_to.window(windows[0]) - wait_elem_text('d2', 'ofanL') - wait_elem_text('d2', 'forehead') - for i in (0, 1): - text = execute_in_page('returnval(divs[arguments[0]].innerText);', i) - assert 'ofanL' not in text - assert 'forehead' not in text diff --git a/test/unit/test_content.py b/test/unit/test_content.py deleted file mode 100644 index 8220160..0000000 --- a/test/unit/test_content.py +++ /dev/null @@ -1,190 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -""" -Haketilo unit tests - main content script -""" - -# This file is part of Haketilo -# -# Copyright (C) 2022 Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -import pytest -import json -from selenium.webdriver.support.ui import WebDriverWait - -from ..script_loader import load_script - -# From: -# https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/contentScripts/register -# it is unclear whether the dynamically-registered content script is guaranteed -# to be always executed after statically-registered ones. We want to test both -# cases, so we'll make the mocked dynamic content script execute before -# content.js on http:// pages and after it on https:// pages. -dynamic_script = \ - '''; - this.haketilo_secret = "abracadabra"; - this.haketilo_pattern_tree = {}; - this.haketilo_default_allow = false; - - if (this.haketilo_content_script_main) - this.haketilo_content_script_main(); - ''' - -content_script = \ - ''' - /* Mock dynamic content script - case 'before'. */ - if (/dynamic_before/.test(document.URL)) { - %s; - } - - /* Place amalgamated content.js here. */ - %s; - - /* Rest of mocks */ - - function mock_decide_policy() { - nonce = "12345"; - return { - allow: false, - mapping: "what-is-programmers-favorite-drinking-place", - payload: {identifier: "foo-bar"}, - nonce, - csp: "prefetch-src 'none'; script-src-attr 'none'; script-src 'nonce-12345'; script-src-elem 'nonce-12345';" - }; - } - - async function mock_payload_error([type, res_id]) { - if (type === "indexeddb_files") - return {error: {haketilo_error_type: "missing", id: res_id}}; - } - - async function mock_payload_ok([type, res_id]) { - if (type === "indexeddb_files") - return {files: [1, 2].map(n => `window.hak_injected_${n} = ${n};`)}; - } - - if (/payload_error/.test(document.URL)) { - browser.runtime.sendMessage = mock_payload_error; - decide_policy = mock_decide_policy; - } else if (/payload_ok/.test(document.URL)) { - browser.runtime.sendMessage = mock_payload_ok; - decide_policy = mock_decide_policy; - } - /* Otherwise, script blocking policy without payload to inject is used. */ - - const data_to_verify = {}; - function data_set(prop, val) { - data_to_verify[prop] = val; - window.wrappedJSObject.data_to_verify = JSON.stringify(data_to_verify); - } - - repo_query_cacher.start = () => data_set("cacher_started", true); - - enforce_blocking = policy => data_set("enforcing", policy); - - browser.runtime.onMessage.addListener = async function (listener_cb) { - await new Promise(cb => setTimeout(cb, 10)); - - /* Mock a good request. */ - const set_good = val => data_set("good_request_result", val); - data_set("good_request_returned", - !!listener_cb(["page_info"], {}, val => set_good(val))); - - /* Mock a bad request. */ - const set_bad = val => data_set("bad_request_result", val); - data_set("bad_request_returned", - !!listener_cb(["???"], {}, val => set_bad(val))); - } - - /* main() call - normally present in content.js, inside '#IF !UNIT_TEST'. */ - main(); - - /* Mock dynamic content script - case 'after'. */ - if (/#dynamic_after/.test(document.URL)) { - %s; - } - - data_set("script_run_without_errors", true); - ''' % (dynamic_script, load_script('content/content.js'), dynamic_script) - -@pytest.mark.ext_data({'content_script': content_script}) -@pytest.mark.usefixtures('webextension') -@pytest.mark.parametrize('target1', ['dynamic_before'])#, 'dynamic_after']) -@pytest.mark.parametrize('target2', [ - 'scripts_blocked', - 'payload_error', - 'payload_ok' -]) -def test_content_unprivileged_page(driver, execute_in_page, target1, target2): - """ - Test functioning of content.js on an page using unprivileged schema (e.g. - 'https://' and not 'about:'). - """ - driver.get(f'https://gotmyowndoma.in/index.html#{target1}-{target2}') - - def get_data(driver): - data = driver.execute_script('return window.data_to_verify;') - return data if 'good_request_result' in data else False - - data = json.loads(WebDriverWait(driver, 10).until(get_data)) - - assert 'gotmyowndoma.in' in data['good_request_result']['url'] - assert 'bad_request_result' not in data - - assert data['good_request_returned'] == True - assert data['bad_request_returned'] == False - - assert data['cacher_started'] == True - - for obj in (data['good_request_result'], data['enforcing']): - assert obj['allow'] == False - - assert 'error' not in data['enforcing'] - - if target2.startswith('payload'): - for obj in (data['good_request_result'], data['enforcing']): - assert obj['payload']['identifier'] == 'foo-bar' - assert 'mapping' in obj - else: - assert 'payload' not in data['enforcing'] - assert 'mapping' not in data['enforcing'] - - assert data['script_run_without_errors'] == True - - def vars_made_by_payload(driver): - vars_values = driver.execute_script( - 'return [1, 2].map(n => window[`hak_injected_${n}`]);' - ) - if vars_values != [None, None]: - return vars_values - - if target2 == 'payload_error': - assert data['good_request_result']['error'] == { - 'haketilo_error_type': 'missing', - 'id': 'foo-bar' - } - elif target2 == 'payload_ok': - vars_values = WebDriverWait(driver, 10).until(vars_made_by_payload) - assert vars_values == [1, 2] - -@pytest.mark.ext_data({'content_script': content_script}) -@pytest.mark.usefixtures('webextension') -@pytest.mark.parametrize('target', ['dynamic_before', 'dynamic_after']) -def test_content_privileged_page(driver, execute_in_page, target): - """ - Test functioning of content.js on an page considered privileged (e.g. a - directory listing at 'file:///'). - """ - driver.get(f'file:///#{target}') - data = json.loads(driver.execute_script('return window.data_to_verify;')) - - assert data == {'script_run_without_errors': True} diff --git a/test/unit/test_default_policy_dialog.py b/test/unit/test_default_policy_dialog.py deleted file mode 100644 index a1c825f..0000000 --- a/test/unit/test_default_policy_dialog.py +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -""" -Haketilo unit tests - default script blocking policy dialog -""" - -# This file is part of Haketilo -# -# Copyright (C) 2022, Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -import pytest - -from ..extension_crafting import ExtraHTML -from ..script_loader import load_script -from .utils import broker_js - -@pytest.mark.ext_data({ - 'background_script': broker_js, - 'extra_html': ExtraHTML( - 'html/default_blocking_policy.html', - { - 'html/default_blocking_policy.js': - 'init_default_policy_dialog();' - } - ), - 'navigate_to': 'html/default_blocking_policy.html' -}) -@pytest.mark.usefixtures('webextension') -def test_default_blocking_policy_dialog(driver, wait_elem_text): - """ - A test case for the dialog that facilitates toggling the default policy of - script blocking. - """ - wait_elem_text('current_policy_span', 'block') - - driver.find_element_by_id('toggle_policy_but').click() - wait_elem_text('current_policy_span', 'allow') - - driver.find_element_by_id('toggle_policy_but').click() - wait_elem_text('current_policy_span', 'block') diff --git a/test/unit/test_dialog.py b/test/unit/test_dialog.py deleted file mode 100644 index 63af79e..0000000 --- a/test/unit/test_dialog.py +++ /dev/null @@ -1,143 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -""" -Haketilo unit tests - showing an error/info/question dalog -""" - -# This file is part of Haketilo -# -# Copyright (C) 2022, Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -import pytest - -from ..extension_crafting import ExtraHTML -from ..script_loader import load_script - -@pytest.mark.ext_data({ - 'extra_html': ExtraHTML('html/dialog.html', {}), - 'navigate_to': 'html/dialog.html' -}) -@pytest.mark.usefixtures('webextension') -def test_dialog_show_close(driver, execute_in_page): - """ - A test case of basic dialog showing/closing. - """ - execute_in_page(load_script('html/dialog.js')) - buts = execute_in_page( - ''' - let cb_calls, call_prom; - const dialog_context = make(() => cb_calls.push("show"), - () => cb_calls.push("hide")); - document.body.append(dialog_context.main_div); - const buts = {}; - for (const but of document.getElementsByTagName("button")) - buts[but.textContent] = but; - returnval(buts); - ''') - - for i, (dialog_function, but_text, hidden, expected_result) in enumerate([ - ('info', 'Ok', ['Yes', 'No'], None), - ('error', 'Ok', ['Yes', 'No'], None), - ('error', None, ['Yes', 'No'], None), - ('loader', None, ['Yes', 'No', 'Ok'], None), - ('ask', 'Yes', ['Ok'], True), - ('ask', None, ['Ok'], None), - ('ask', 'No', ['Ok'], False) - ]): - cb_calls, is_shown = execute_in_page( - f''' - cb_calls = []; - call_prom = {dialog_function}(dialog_context, - `sample_text_${{arguments[0]}}`); - returnval([cb_calls, dialog_context.shown]); - ''', - i) - assert cb_calls == ['show'] - assert is_shown == True - - page_source = driver.page_source - assert f'sample_text_{i}' in page_source - assert f'sample_text_{i - 1}' not in page_source - - # Verify the right buttons are displayed. - for text, but in buts.items(): - if text in hidden: - assert not but.is_displayed() - # Verify clicking a hidden button does nothing. - execute_in_page('buts[arguments[0]].click();', text) - assert execute_in_page('returnval(cb_calls);') == cb_calls - else: - assert but.is_displayed() - - if but_text is None: - execute_in_page('close_dialog(dialog_context);') - else: - buts[but_text].click() - - cb_calls, result, is_shown = execute_in_page( - '''{ - const values_cb = r => [cb_calls, r, dialog_context.shown]; - returnval(call_prom.then(values_cb)); - }''') - assert cb_calls == ['show', 'hide'] - assert result == expected_result - assert is_shown == False - -@pytest.mark.ext_data({ - 'extra_html': ExtraHTML('html/dialog.html', {}), - 'navigate_to': 'html/dialog.html' -}) -@pytest.mark.usefixtures('webextension') -def test_dialog_queue(driver, execute_in_page): - """ - A test case of queuing dialog display operations. - """ - execute_in_page(load_script('html/dialog.js')) - execute_in_page( - ''' - let cb_calls = [], call_proms = []; - const dialog_context = make(() => cb_calls.push("show"), - () => cb_calls.push("hide")); - document.body.append(dialog_context.main_div); - ''') - - buts = driver.find_elements_by_tag_name('button') - buts = dict([(but.text, but) for but in buts]) - - for i in range(5): - cb_calls, is_shown, msg_elem = execute_in_page( - ''' - call_proms.push(ask(dialog_context, "somequestion" + arguments[0])); - returnval([cb_calls, dialog_context.shown, dialog_context.msg]); - ''', - i) - assert cb_calls == ['show'] - assert is_shown == True - assert msg_elem.text == 'somequestion0' - - for i in range(5): - buts['Yes' if i & 1 else 'No'].click() - cb_calls, is_shown, msg_elem, result = execute_in_page( - '''{ - const values_cb = - r => [cb_calls, dialog_context.shown, dialog_context.msg, r]; - returnval(call_proms.splice(0, 1)[0].then(values_cb)); - }''') - if i < 4: - assert cb_calls == ['show'] - assert is_shown == True - assert msg_elem.text == f'somequestion{i + 1}' - else: - assert cb_calls == ['show', 'hide'] - assert is_shown == False - - assert result == bool(i & 1) diff --git a/test/unit/test_indexeddb.py b/test/unit/test_indexeddb.py deleted file mode 100644 index c2d5427..0000000 --- a/test/unit/test_indexeddb.py +++ /dev/null @@ -1,490 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -""" -Haketilo unit tests - IndexedDB access -""" - -# This file is part of Haketilo -# -# Copyright (C) 2021, 2022 Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -import pytest -import json -from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC -from selenium.common.exceptions import WebDriverException - -from ..script_loader import load_script -from .utils import * - -# Sample resource definitions. They'd normally contain more fields but here we -# use simplified versions. - -def make_sample_resource(): - return { - 'source_copyright': [ - sample_file_ref('report.spdx'), - sample_file_ref('LICENSES/somelicense.txt') - ], - 'type': 'resource', - 'identifier': 'helloapple', - 'scripts': [sample_file_ref('hello.js'), sample_file_ref('bye.js')] - } - -def make_sample_mapping(): - return { - 'source_copyright': [ - sample_file_ref('report.spdx'), - sample_file_ref('README.md') - ], - 'type': 'mapping', - 'identifier': 'helloapple' - } - -@pytest.mark.get_page('https://gotmyowndoma.in') -def test_haketilodb_item_modifications(driver, execute_in_page): - """ - indexeddb.js facilitates operating on Haketilo's internal database. - Verify database operations on mappings/resources work properly. - """ - execute_in_page(load_script('common/indexeddb.js')) - mock_broadcast(execute_in_page) - - # Start with no database. - clear_indexeddb(execute_in_page) - - sample_item = make_sample_resource() - sample_item['source_copyright'][0]['extra_prop'] = True - - execute_in_page( - '''{ - const promise = start_items_transaction(["resource"], arguments[1]) - .then(ctx => save_item(arguments[0], ctx).then(() => ctx)) - .then(finalize_transaction); - returnval(promise); - }''', - sample_item, {'sha256': sample_files_by_sha256}) - - database_contents = get_db_contents(execute_in_page) - - assert len(database_contents['file']) == 4 - assert all([sample_files_by_sha256[file['sha256']] == file['contents'] - for file in database_contents['file']]) - assert all([len(file) == 2 for file in database_contents['file']]) - - assert len(database_contents['file_uses']) == 4 - assert all([uses['uses'] == 1 for uses in database_contents['file_uses']]) - assert set([uses['sha256'] for uses in database_contents['file_uses']]) \ - == set([file['sha256'] for file in database_contents['file']]) - - assert database_contents['mapping'] == [] - assert database_contents['resource'] == [sample_item] - - # See if trying to add an item without providing all its files ends in an - # exception and aborts the transaction as it should. - sample_item['scripts'].append(sample_file_ref('combined.js')) - incomplete_files = {**sample_files_by_sha256} - incomplete_files.pop(sample_files['combined.js']['sha256']) - exception = execute_in_page( - '''{ - const args = arguments; - async function try_add_item() - { - const context = - await start_items_transaction(["resource"], args[1]); - try { - await save_item(args[0], context); - await finalize_transaction(context); - return; - } catch(e) { - return e; - } - } - returnval(try_add_item()); - }''', - sample_item, {'sha256': incomplete_files}) - - previous_database_contents = database_contents - database_contents = get_db_contents(execute_in_page) - - assert 'file not present' in exception - for key, val in database_contents.items(): - keyfun = lambda item: item.get('sha256') or item['identifier'] - assert sorted(previous_database_contents[key], key=keyfun) \ - == sorted(val, key=keyfun) - - # See if adding another item that partially uses first's files works OK. - sample_item = make_sample_mapping() - database_contents = execute_in_page( - '''{ - const promise = start_items_transaction(["mapping"], arguments[1]) - .then(ctx => save_item(arguments[0], ctx).then(() => ctx)) - .then(finalize_transaction); - returnval(promise); - }''', - sample_item, {'sha256': sample_files_by_sha256}) - - database_contents = get_db_contents(execute_in_page) - - names = ['README.md', 'report.spdx', 'LICENSES/somelicense.txt', 'hello.js', - 'bye.js'] - sample_files_list = [sample_files[name] for name in names] - uses_list = [1, 2, 1, 1, 1] - - uses = dict([(uses['sha256'], uses['uses']) - for uses in database_contents['file_uses']]) - assert uses == dict([(file['sha256'], nr) - for file, nr in zip(sample_files_list, uses_list)]) - - files = dict([(file['sha256'], file['contents']) - for file in database_contents['file']]) - assert files == dict([(file['sha256'], file['contents']) - for file in sample_files_list]) - - del database_contents['resource'][0]['source_copyright'][0]['extra_prop'] - assert database_contents['resource'] == [make_sample_resource()] - assert database_contents['mapping'] == [sample_item] - - # Try removing the items to get an empty database again. - results = [None, None] - for i, item_type in enumerate(['resource', 'mapping']): - execute_in_page( - f'''{{ - const remover = remove_{item_type}; - const promise = - start_items_transaction(["{item_type}"], {{}}) - .then(ctx => remover('helloapple', ctx).then(() => ctx)) - .then(finalize_transaction); - returnval(promise); - }}''') - - results[i] = get_db_contents(execute_in_page) - - names = ['README.md', 'report.spdx'] - sample_files_list = [sample_files[name] for name in names] - uses_list = [1, 1] - - uses = dict([(uses['sha256'], uses['uses']) - for uses in results[0]['file_uses']]) - assert uses == dict([(file['sha256'], 1) for file in sample_files_list]) - - files = dict([(file['sha256'], file['contents']) - for file in results[0]['file']]) - assert files == dict([(file['sha256'], file['contents']) - for file in sample_files_list]) - - assert results[0]['resource'] == [] - assert results[0]['mapping'] == [sample_item] - - assert results[1] == dict([(key, []) for key in results[0].keys()]) - - # Try initializing an empty database with sample initial data object. - sample_resource = make_sample_resource() - sample_mapping = make_sample_mapping() - initial_data = { - 'resource': { - 'helloapple': { - '1.12': sample_resource, - '0.9': 'something_that_should_get_ignored', - '1': 'something_that_should_get_ignored', - '1.1': 'something_that_should_get_ignored', - '1.11.1': 'something_that_should_get_ignored', - } - }, - 'mapping': { - 'helloapple': { - '0.1.1': sample_mapping - } - }, - 'file': { - 'sha256': sample_files_by_sha256 - } - } - - clear_indexeddb(execute_in_page) - execute_in_page('initial_data = arguments[0];', initial_data) - database_contents = get_db_contents(execute_in_page) - - assert database_contents['resource'] == [sample_resource] - assert database_contents['mapping'] == [sample_mapping] - -@pytest.mark.get_page('https://gotmyowndoma.in') -def test_haketilodb_settings(driver, execute_in_page): - """ - indexeddb.js facilitates operating on Haketilo's internal database. - Verify assigning/retrieving values of simple "setting" item works properly. - """ - execute_in_page(load_script('common/indexeddb.js')) - mock_broadcast(execute_in_page) - - # Start with no database. - clear_indexeddb(execute_in_page) - - assert get_db_contents(execute_in_page)['setting'] == [] - - assert execute_in_page('returnval(get_setting("option15"));') == None - - execute_in_page('returnval(set_setting("option15", "disable"));') - assert execute_in_page('returnval(get_setting("option15"));') == 'disable' - - execute_in_page('returnval(set_setting("option15", "enable"));') - assert execute_in_page('returnval(get_setting("option15"));') == 'enable' - -@pytest.mark.get_page('https://gotmyowndoma.in') -def test_haketilodb_allowing(driver, execute_in_page): - """ - indexeddb.js facilitates operating on Haketilo's internal database. - Verify changing the "blocking" configuration for a URL works properly. - """ - execute_in_page(load_script('common/indexeddb.js')) - mock_broadcast(execute_in_page) - - # Start with no database. - clear_indexeddb(execute_in_page) - - assert get_db_contents(execute_in_page)['blocking'] == [] - - def run_with_sample_url(expr): - return execute_in_page(f'returnval({expr});', 'https://example.com/**') - - assert None == run_with_sample_url('get_allowing(arguments[0])') - - run_with_sample_url('set_disallowed(arguments[0])') - assert False == run_with_sample_url('get_allowing(arguments[0])') - - run_with_sample_url('set_allowed(arguments[0])') - assert True == run_with_sample_url('get_allowing(arguments[0])') - - run_with_sample_url('set_default_allowing(arguments[0])') - assert None == run_with_sample_url('get_allowing(arguments[0])') - -@pytest.mark.get_page('https://gotmyowndoma.in') -def test_haketilodb_repos(driver, execute_in_page): - """ - indexeddb.js facilitates operating on Haketilo's internal database. - Verify operations on repositories list work properly. - """ - execute_in_page(load_script('common/indexeddb.js')) - mock_broadcast(execute_in_page) - - # Start with no database. - clear_indexeddb(execute_in_page) - - assert get_db_contents(execute_in_page)['repo'] == [] - - sample_urls = ['https://hdrlla.example.com/', 'https://hdrlla.example.org'] - - assert [] == execute_in_page('returnval(get_repos());') - - execute_in_page('returnval(set_repo(arguments[0]));', sample_urls[0]) - assert [sample_urls[0]] == execute_in_page('returnval(get_repos());') - - execute_in_page('returnval(set_repo(arguments[0]));', sample_urls[1]) - assert set(sample_urls) == set(execute_in_page('returnval(get_repos());')) - - execute_in_page('returnval(del_repo(arguments[0]));', sample_urls[0]) - assert [sample_urls[1]] == execute_in_page('returnval(get_repos());') - -test_page_html = ''' - - - - -''' - -@pytest.mark.ext_data({ - 'background_script': broker_js, - 'test_page': test_page_html, - 'extra_files': { - 'testpage.js': lambda: load_script('common/indexeddb.js') - } -}) -@pytest.mark.usefixtures('webextension') -def test_haketilodb_track(driver, execute_in_page, wait_elem_text): - """ - Verify IndexedDB object change notifications are correctly broadcasted - through extension's background script and allow for object store contents - to be tracked in any execution context. - """ - # Let's open the same extension's test page in a second window. Window 1 - # will be used to make changes to IndexedDB and window 0 to "track" those - # changes. - driver.execute_script('window.open(window.location.href, "_blank");') - WebDriverWait(driver, 10).until(lambda d: len(d.window_handles) == 2) - windows = [*driver.window_handles] - - # Create elements that will have tracked data inserted under them. - driver.switch_to.window(windows[0]) - execute_in_page( - ''' - for (const store_name of trackable) { - const h2 = document.createElement("h2"); - h2.innerText = store_name; - document.body.append(h2); - - const ul = document.createElement("ul"); - ul.id = store_name; - document.body.append(ul); - } - ''') - - # Mock initial_data. - sample_resource = make_sample_resource() - sample_mapping = make_sample_mapping() - initial_data = { - 'resource': { - 'helloapple': { - '1.0': sample_resource - } - }, - 'mapping': { - 'helloapple': { - '0.1.1': sample_mapping - } - }, - 'file': { - 'sha256': sample_files_by_sha256 - } - } - driver.switch_to.window(windows[1]) - execute_in_page('initial_data = arguments[0];', initial_data) - execute_in_page('returnval(set_setting("option15", "123"));') - execute_in_page('returnval(set_repo("https://hydril.la"));') - execute_in_page('returnval(set_disallowed("file:///*"));') - - # See if track.*() functions properly return the already-existing items. - driver.switch_to.window(windows[0]) - execute_in_page( - ''' - function update_item(store_name, change) - { - const elem_id = `${store_name}_${change.key}`; - let elem = document.getElementById(elem_id); - elem = elem || document.createElement("li"); - elem.id = elem_id; - elem.innerText = JSON.stringify(change.new_val); - document.getElementById(store_name).append(elem); - if (change.new_val === undefined) - elem.remove(); - } - - let resource_tracking, resource_items, mapping_tracking, mapping_items; - - async function start_reporting() - { - const props = new Map(stores.map(([sn, opt]) => [sn, opt.keyPath])); - for (const store_name of trackable) { - [tracking, items] = - await track[store_name](ch => update_item(store_name, ch)); - const prop = props.get(store_name); - for (const item of items) - update_item(store_name, {key: item[prop], new_val: item}); - } - } - - returnval(start_reporting()); - ''') - - item_counts = execute_in_page( - '''{ - const childcount = id => document.getElementById(id).childElementCount; - returnval(trackable.map(childcount)); - }''') - assert item_counts == [1 for _ in item_counts] - for elem_id, json_value in [ - ('resource_helloapple', sample_resource), - ('mapping_helloapple', sample_mapping), - ('setting_option15', {'name': 'option15', 'value': '123'}), - ('repo_https://hydril.la', {'url': 'https://hydril.la'}), - ('blocking_file:///*', {'pattern': 'file:///*', 'allow': False}) - ]: - assert json.loads(driver.find_element_by_id(elem_id).text) == json_value - - # See if item additions get tracked properly. - driver.switch_to.window(windows[1]) - sample_resource2 = make_sample_resource() - sample_resource2['identifier'] = 'helloapple-copy' - sample_mapping2 = make_sample_mapping() - sample_mapping2['identifier'] = 'helloapple-copy' - sample_data = { - 'resource': { - 'helloapple-copy': { - '1.0': sample_resource2 - } - }, - 'mapping': { - 'helloapple-copy': { - '0.1.1': sample_mapping2 - } - }, - 'file': { - 'sha256': sample_files_by_sha256 - }, - 'repo': [ - 'https://hydril2.la/' - ] - } - execute_in_page('returnval(save_items(arguments[0]));', sample_data) - execute_in_page('returnval(set_setting("option22", "abc"));') - execute_in_page('returnval(set_repo("https://hydril3.la/"));') - execute_in_page('returnval(set_allowed("ftp://a.bc/"));') - - driver.switch_to.window(windows[0]) - driver.implicitly_wait(10) - for elem_id, json_value in [ - ('resource_helloapple-copy', sample_resource2), - ('mapping_helloapple-copy', sample_mapping2), - ('setting_option22', {'name': 'option22', 'value': 'abc'}), - ('repo_https://hydril2.la/', {'url': 'https://hydril2.la/'}), - ('repo_https://hydril3.la/', {'url': 'https://hydril3.la/'}), - ('blocking_ftp://a.bc/', {'pattern': 'ftp://a.bc/', 'allow': True}) - ]: - assert json.loads(driver.find_element_by_id(elem_id).text) == json_value - driver.implicitly_wait(0) - - # See if item deletions/modifications get tracked properly. - driver.switch_to.window(windows[1]) - execute_in_page( - '''{ - async function change_remove_items() - { - const store_names = ["resource", "mapping"]; - const ctx = await start_items_transaction(store_names, {}); - await remove_resource("helloapple", ctx); - await remove_mapping("helloapple-copy", ctx); - await finalize_transaction(ctx); - await set_setting("option22", null); - await del_repo("https://hydril.la"); - await set_default_allowing("file:///*"); - await set_disallowed("ftp://a.bc/"); - } - returnval(change_remove_items()); - }''') - - removed_ids = ['mapping_helloapple-copy', 'resource_helloapple', - 'repo_https://hydril.la', 'blocking_file:///*'] - def condition_items_absent_and_changed(driver): - for id in removed_ids: - try: - driver.find_element_by_id(id) - return False - except WebDriverException: - pass - - option_text = driver.find_element_by_id('setting_option22').text - blocking_text = driver.find_element_by_id('blocking_ftp://a.bc/').text - return (json.loads(option_text)['value'] == None and - json.loads(blocking_text)['allow'] == False) - - driver.switch_to.window(windows[0]) - WebDriverWait(driver, 10).until(condition_items_absent_and_changed) diff --git a/test/unit/test_indexeddb_files_server.py b/test/unit/test_indexeddb_files_server.py deleted file mode 100644 index 6ddfba8..0000000 --- a/test/unit/test_indexeddb_files_server.py +++ /dev/null @@ -1,171 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -""" -Haketilo unit tests - serving indexeddb resource script files to content scripts -""" - -# This file is part of Haketilo -# -# Copyright (C) 2021,2022 Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -import pytest -import copy -from uuid import uuid4 -from selenium.webdriver.support.ui import WebDriverWait - -from ..script_loader import load_script -from .utils import * - -""" -How many test resources we're going to have. -""" -count = 15 - -sample_files_list = [(f'file_{n}_{i}', f'contents {n} {i}') - for n in range(count) for i in range(2)] - -sample_files = dict(sample_files_list) - -sample_files, sample_files_by_sha256 = make_sample_files(sample_files) - -def make_sample_resource_with_deps(n): - resource = make_sample_resource(with_files=False) - - resource['identifier'] = f'res-{n}' - resource['dependencies'] = [{'identifier': f'res-{m}'} - for m in range(max(n - 4, 0), n)] - resource['scripts'] = [sample_file_ref(f'file_{n}_{i}', sample_files) - for i in range(2)] - - return resource - -resources = [make_sample_resource_with_deps(n) for n in range(count)] - -sample_data = { - 'resource': sample_data_dict(resources), - 'mapping': {}, - 'file': { - 'sha256': sample_files_by_sha256 - } -} - -def prepare_test_page(initial_indexeddb_data, execute_in_page): - js = load_script('background/indexeddb_files_server.js', - code_to_add='#IMPORT common/broadcast.js') - execute_in_page(js) - - mock_broadcast(execute_in_page) - clear_indexeddb(execute_in_page) - - execute_in_page( - ''' - let registered_listener; - const new_addListener = cb => registered_listener = cb; - - browser = {runtime: {onMessage: {addListener: new_addListener}}}; - - haketilodb.save_items(arguments[0]); - - start(); - ''', - initial_indexeddb_data) - -@pytest.mark.get_page('https://gotmyowndoma.in') -def test_indexeddb_files_server_normal_usage(driver, execute_in_page): - """ - Test querying resource files (with resource dependency resolution) - from IndexedDB and serving them in messages to content scripts. - """ - prepare_test_page(sample_data, execute_in_page) - - # Verify other types of messages are ignored. - function_returned_value = execute_in_page( - ''' - returnval(registered_listener(["???"], {}, - () => location.reload())); - ''') - assert function_returned_value == None - - # Verify single resource's files get properly resolved. - function_returned_value = execute_in_page( - ''' - var result_cb, contents_prom = new Promise(cb => result_cb = cb); - - returnval(registered_listener(["indexeddb_files", "res-0"], - {}, result_cb)); - ''') - assert function_returned_value == True - - assert execute_in_page('returnval(contents_prom);') == \ - {'files': [tuple[1] for tuple in sample_files_list[0:2]]} - - # Verify multiple resources' files get properly resolved. - function_returned_value = execute_in_page( - ''' - var result_cb, contents_prom = new Promise(cb => result_cb = cb); - - returnval(registered_listener(["indexeddb_files", arguments[0]], - {}, result_cb)); - ''', - f'res-{count - 1}') - assert function_returned_value == True - - assert execute_in_page('returnval(contents_prom);') == \ - {'files': [tuple[1] for tuple in sample_files_list]} - -@pytest.mark.get_page('https://gotmyowndoma.in') -@pytest.mark.parametrize('error', [ - 'missing', - 'circular', - 'db', - 'other' -]) -def test_indexeddb_files_server_errors(driver, execute_in_page, error): - """ - Test reporting of errors when querying resource files (with resource - dependency resolution) from IndexedDB and serving them in messages to - content scripts. - """ - sample_data_copy = copy.deepcopy(sample_data) - - if error == 'missing': - del sample_data_copy['resource']['res-3'] - elif error == 'circular': - res3_defs = sample_data_copy['resource']['res-3'].values() - next(iter(res3_defs))['dependencies'].append({'identifier': 'res-8'}) - - prepare_test_page(sample_data_copy, execute_in_page) - - if error == 'db': - execute_in_page('haketilodb.idb_get = t => t.onerror("oooops");') - elif error == 'other': - execute_in_page('haketilodb.idb_get = () => {throw "oooops"};') - - response = execute_in_page( - ''' - var result_cb, contents_prom = new Promise(cb => result_cb = cb); - - registered_listener(["indexeddb_files", arguments[0]], - {}, result_cb); - - returnval(contents_prom); - ''', - f'res-{count - 1}') - - assert response['error']['haketilo_error_type'] == error - - if error == 'missing': - assert response['error']['id'] == 'res-3' - elif error == 'circular': - assert response['error']['id'] in ('res-3', 'res-8') - elif error not in ('db', 'other'): - raise Exception('made a typo in test function params?') diff --git a/test/unit/test_install.py b/test/unit/test_install.py deleted file mode 100644 index f4bc483..0000000 --- a/test/unit/test_install.py +++ /dev/null @@ -1,423 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -""" -Haketilo unit tests - item installation dialog -""" - -# This file is part of Haketilo -# -# Copyright (C) 2022 Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -import pytest -import json -from selenium.webdriver.support.ui import WebDriverWait - -from ..extension_crafting import ExtraHTML -from ..script_loader import load_script -from .utils import * - -def setup_view(driver, execute_in_page): - mock_cacher(execute_in_page) - - execute_in_page(load_script('html/install.js')) - container_ids, containers_objects = execute_in_page( - ''' - const cb_calls = []; - const install_view = new InstallView(0, - () => cb_calls.push("show"), - () => cb_calls.push("hide")); - document.body.append(install_view.main_div); - const ets = () => install_view.item_entries; - const shw = slice => [cb_calls.slice(slice || 0), install_view.shown]; - returnval([container_ids, container_ids.map(cid => install_view[cid])]); - ''') - - containers = dict(zip(container_ids, containers_objects)) - - def assert_container_displayed(container_id): - for cid, cobj in zip(container_ids, containers_objects): - assert (cid == container_id) == cobj.is_displayed() - - return containers, assert_container_displayed - -install_ext_data = { - 'background_script': broker_js, - 'extra_html': ExtraHTML('html/install.html', {}), - 'navigate_to': 'html/install.html' -} - -@pytest.mark.ext_data(install_ext_data) -@pytest.mark.usefixtures('webextension') -@pytest.mark.parametrize('complex_variant', [False, True]) -def test_install_normal_usage(driver, execute_in_page, complex_variant): - """ - Test of the normal package installation procedure with one mapping and, - depending on parameter, one or many resources. - """ - containers, assert_container_displayed = setup_view(driver, execute_in_page) - - assert execute_in_page('returnval(shw());') == [[], False] - - if complex_variant: - # The resource/mapping others depend on. - root_id = 'abcd-defg-ghij' - root_resource_id = f'resource_{root_id}' - root_mapping_id = f'mapping_{root_id}' - # Those ids are used to check the alphabetical ordering. - resource_ids = [f'resource_{letters}' for letters in ( - 'a', 'abcd', root_id, 'b', 'c', - 'd', 'defg', 'e', 'f', - 'g', 'ghij', 'h', 'i', 'j' - )] - files_count = 9 - else: - root_resource_id = f'resource_a' - root_mapping_id = f'mapping_a' - resource_ids = [root_resource_id] - files_count = 0 - - # Preview the installation of a resource, show resource's details, close - # the details and cancel installation. - execute_in_page('returnval(install_view.show(...arguments));', - 'https://hydril.la/', 'resource', root_resource_id) - - assert execute_in_page('returnval(shw());') == [['show'], True] - assert f'{root_resource_id}-2021.11.11-1'\ - in containers['install_preview'].text - assert_container_displayed('install_preview') - - entries = execute_in_page('returnval(ets().map(e => e.main_li.innerText));') - assert len(entries) == len(resource_ids) - # Verify alphabetical ordering. - assert all([id in text for id, text in zip(resource_ids, entries)]) - - assert not execute_in_page('returnval(ets()[0].old_ver);').is_displayed() - execute_in_page('returnval(ets()[0].details_but);').click() - assert 'resource_a' in containers['resource_preview_container'].text - assert_container_displayed('resource_preview_container') - - execute_in_page('returnval(install_view.resource_back_but);').click() - assert_container_displayed('install_preview') - - assert execute_in_page('returnval(shw());') == [['show'], True] - execute_in_page('returnval(install_view.cancel_but);').click() - assert execute_in_page('returnval(shw());') == [['show', 'hide'], False] - - # Preview the installation of a mapping and a resource, show mapping's - # details, close the details and commit the installation. - execute_in_page('returnval(install_view.show(...arguments));', - 'https://hydril.la/', 'mapping', - root_mapping_id, [2022, 5, 10]) - - assert execute_in_page('returnval(shw(2));') == [['show'], True] - assert_container_displayed('install_preview') - - entries = execute_in_page('returnval(ets().map(e => e.main_li.innerText));') - assert len(entries) == len(resource_ids) + 1 - assert f'{root_mapping_id}-2022.5.10' in entries[0] - # Verify alphabetical ordering. - assert all([id in text for id, text in zip(resource_ids, entries[1:])]) - - assert not execute_in_page('returnval(ets()[0].old_ver);').is_displayed() - execute_in_page('returnval(ets()[0].details_but);').click() - assert root_mapping_id in containers['mapping_preview_container'].text - assert_container_displayed('mapping_preview_container') - - execute_in_page('returnval(install_view.mapping_back_but);').click() - assert_container_displayed('install_preview') - - execute_in_page('returnval(install_view.install_but);').click() - installed = lambda d: 'ly installed!' in containers['dialog_container'].text - WebDriverWait(driver, 10).until(installed) - - assert execute_in_page('returnval(shw(2));') == [['show'], True] - execute_in_page('returnval(install_view.dialog_ctx.ok_but);').click() - assert execute_in_page('returnval(shw(2));') == [['show', 'hide'], False] - - # Verify the install - db_contents = get_db_contents(execute_in_page) - for item_type, ids in \ - [('mapping', {root_mapping_id}), ('resource', set(resource_ids))]: - assert set([it['identifier'] for it in db_contents[item_type]]) == ids - - assert all([len(db_contents[store]) == files_count - for store in ('file', 'file_uses')]) - - # Update the installed mapping to a newer version. - execute_in_page('returnval(install_view.show(...arguments));', - 'https://hydril.la/', 'mapping', root_mapping_id) - assert execute_in_page('returnval(shw(4));') == [['show'], True] - # resources are already in the newest versions, hence they should not appear - # in the install preview list. - assert execute_in_page('returnval(ets().length);') == 1 - # Mapping's version update information should be displayed. - assert execute_in_page('returnval(ets()[0].old_ver);').is_displayed() - execute_in_page('returnval(install_view.install_but);').click() - - WebDriverWait(driver, 10).until(installed) - - assert execute_in_page('returnval(shw(4));') == [['show'], True] - execute_in_page('returnval(install_view.dialog_ctx.ok_but);').click() - assert execute_in_page('returnval(shw(4));') == [['show', 'hide'], False] - - # Verify the newer version install. - old_db_contents, db_contents = db_contents, get_db_contents(execute_in_page) - old_db_contents['mapping'][0]['version'][-1] += 1 - assert db_contents['mapping'] == old_db_contents['mapping'] - - # All items are up to date - verify dialog is instead shown in this case. - execute_in_page('install_view.show(...arguments);', - 'https://hydril.la/', 'mapping', root_mapping_id) - - fetched = lambda d: 'Fetching ' not in containers['dialog_container'].text - WebDriverWait(driver, 10).until(fetched) - - assert 'Nothing to do - packages already installed.' \ - in containers['dialog_container'].text - assert_container_displayed('dialog_container') - - assert execute_in_page('returnval(shw(6));') == [['show'], True] - execute_in_page('returnval(install_view.dialog_ctx.ok_but);').click() - assert execute_in_page('returnval(shw(6));') == [['show', 'hide'], False] - -@pytest.mark.ext_data(install_ext_data) -@pytest.mark.usefixtures('webextension') -@pytest.mark.parametrize('message', [ - 'fetching_data', - 'failure_to_communicate_sendmessage', - 'HTTP_code_item', - 'invalid_JSON', - 'newer_API_version', - 'invalid_response_format', - 'indexeddb_error_item', - 'installing', - 'indexeddb_error_file_uses', - 'failure_to_communicate_fetch', - 'HTTP_code_file', - 'not_valid_text', - 'sha256_mismatch', - 'indexeddb_error_write' -]) -def test_install_dialogs(driver, execute_in_page, message): - """ - Test of various error and loading messages used in install view. - """ - containers, assert_container_displayed = setup_view(driver, execute_in_page) - - def dlg_buts(): - return execute_in_page( - '''{ - const dlg = install_view.dialog_ctx; - const ids = ['ask_buts', 'conf_buts']; - returnval(ids.filter(id => !dlg[id].classList.contains("hide"))); - }''') - - def dialog_txt(): - return execute_in_page( - 'returnval(install_view.dialog_ctx.msg.textContent);' - ) - - def assert_dlg(awaited_buttons, expected_msg, hides_install_view=True, - button_to_click='ok_but'): - WebDriverWait(driver, 10).until(lambda d: dlg_buts() == awaited_buttons) - - assert expected_msg == dialog_txt() - - execute_in_page( - f'returnval(install_view.dialog_ctx.{button_to_click});' - ).click() - - if hides_install_view: - assert execute_in_page('returnval(shw());') == \ - [['show', 'hide'], False] - - if message == 'fetching_data': - execute_in_page( - ''' - browser.tabs.sendMessage = () => new Promise(cb => {}); - install_view.show(...arguments); - ''', - 'https://hydril.la/', 'mapping', 'mapping_a') - - assert dlg_buts() == [] - assert dialog_txt() == 'Fetching data from repository...' - elif message == 'failure_to_communicate_sendmessage': - execute_in_page( - ''' - browser.tabs.sendMessage = () => Promise.resolve({error: "sth"}); - install_view.show(...arguments); - ''', - 'https://hydril.la/', 'mapping', 'mapping_a') - - assert_dlg(['conf_buts'], 'Failure to communicate with repository :(') - elif message == 'HTTP_code_item': - execute_in_page( - ''' - const response = {ok: false, status: 404}; - browser.tabs.sendMessage = () => Promise.resolve(response); - install_view.show(...arguments); - ''', - 'https://hydril.la/', 'mapping', 'mapping_a') - - assert_dlg(['conf_buts'], 'Repository sent HTTP code 404 :(') - elif message == 'invalid_JSON': - execute_in_page( - ''' - const response = {ok: true, status: 200, error_json: "sth"}; - browser.tabs.sendMessage = () => Promise.resolve(response); - install_view.show(...arguments); - ''', - 'https://hydril.la/', 'mapping', 'mapping_a') - - assert_dlg(['conf_buts'], "Repository's response is not valid JSON :(") - elif message == 'newer_API_version': - execute_in_page( - ''' - const response = { - ok: true, - status: 200, - json: {$schema: "https://hydrilla.koszko.org/schemas/api_mapping_description-2.1.schema.json"} - }; - browser.tabs.sendMessage = () => Promise.resolve(response); - install_view.show(...arguments); - ''', - 'https://hydril.la/', 'mapping', 'somemapping', [2, 1]) - - assert_dlg(['conf_buts'], - 'Mapping somemapping-2.1 was served using unsupported Hydrilla API version. You might need to update Haketilo.') - elif message == 'invalid_response_format': - execute_in_page( - ''' - const response = { - ok: true, - status: 200, - /* $schema is not a string as it should be. */ - json: {$schema: null} - }; - browser.tabs.sendMessage = () => Promise.resolve(response); - install_view.show(...arguments); - ''', - 'https://hydril.la/', 'resource', 'someresource') - - assert_dlg(['conf_buts'], - 'Resource someresource was served using a nonconforming response format.') - elif message == 'indexeddb_error_item': - execute_in_page( - ''' - haketilodb.idb_get = () => {throw "some error";}; - install_view.show(...arguments); - ''', - 'https://hydril.la/', 'mapping', 'mapping_a') - - assert_dlg(['conf_buts'], - "Error accessing Haketilo's internal database :(") - elif message == 'installing': - execute_in_page( - ''' - haketilodb.save_items = () => new Promise(() => {}); - returnval(install_view.show(...arguments)); - ''', - 'https://hydril.la/', 'mapping', 'mapping_b') - - execute_in_page('returnval(install_view.install_but);').click() - - assert dlg_buts() == [] - assert dialog_txt() == 'Installing...' - elif message == 'indexeddb_error_file_uses': - execute_in_page( - ''' - const old_idb_get = haketilodb.idb_get; - haketilodb.idb_get = function(transaction, store_name, identifier) { - if (store_name === "file_uses") - throw "some error"; - return old_idb_get(...arguments); - } - returnval(install_view.show(...arguments)); - ''', - 'https://hydril.la/', 'mapping', 'mapping_b') - - execute_in_page('returnval(install_view.install_but);').click() - - assert_dlg(['conf_buts'], - "Error accessing Haketilo's internal database :(") - elif message == 'failure_to_communicate_fetch': - execute_in_page( - ''' - fetch = () => {throw "some error";}; - returnval(install_view.show(...arguments)); - ''', - 'https://hydril.la/', 'mapping', 'mapping_b') - - execute_in_page('returnval(install_view.install_but);').click() - - assert_dlg(['conf_buts'], - 'Failure to communicate with repository :(') - elif message == 'HTTP_code_file': - execute_in_page( - ''' - fetch = () => Promise.resolve({ok: false, status: 400}); - returnval(install_view.show(...arguments)); - ''', - 'https://hydril.la/', 'mapping', 'mapping_b') - - execute_in_page('returnval(install_view.install_but);').click() - - assert_dlg(['conf_buts'], 'Repository sent HTTP code 400 :(') - elif message == 'not_valid_text': - execute_in_page( - ''' - const err = () => {throw "some error";}; - fetch = () => Promise.resolve({ok: true, status: 200, text: err}); - returnval(install_view.show(...arguments)); - ''', - 'https://hydril.la/', 'mapping', 'mapping_b') - - execute_in_page('returnval(install_view.install_but);').click() - - assert_dlg(['conf_buts'], "Repository's response is not valid text :(") - elif message == 'sha256_mismatch': - execute_in_page( - ''' - let old_fetch = fetch, url_used; - fetch = async function(url) { - url_used = url; - const response = await old_fetch(...arguments); - const text = () => response.text().then(t => t + ":d"); - return {ok: response.ok, status: response.status, text}; - } - returnval(install_view.show(...arguments)); - ''', - 'https://hydril.la/', 'mapping', 'mapping_b') - - execute_in_page('returnval(install_view.install_but);').click() - - get_url_used = lambda d: execute_in_page('returnval(url_used);') - url_used = WebDriverWait(driver, 10).until(get_url_used) - print ((url_used,)) - - assert dlg_buts() == ['conf_buts'] - assert dialog_txt() == \ - f'{url_used} served a file with different SHA256 cryptographic sum :(' - elif message == 'indexeddb_error_write': - execute_in_page( - ''' - haketilodb.save_items = () => {throw "some error";}; - returnval(install_view.show(...arguments)); - ''', - 'https://hydril.la/', 'mapping', 'mapping_b') - - execute_in_page('returnval(install_view.install_but);').click() - - assert_dlg(['conf_buts'], - "Error writing to Haketilo's internal database :(") - else: - raise Exception('made a typo in test function params?') diff --git a/test/unit/test_item_list.py b/test/unit/test_item_list.py deleted file mode 100644 index 35ed1d5..0000000 --- a/test/unit/test_item_list.py +++ /dev/null @@ -1,280 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -""" -Haketilo unit tests - displaying list of resources/mappings -""" - -# This file is part of Haketilo -# -# Copyright (C) 2022, Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -import pytest -from selenium.webdriver.support.ui import WebDriverWait - -from ..extension_crafting import ExtraHTML -from ..script_loader import load_script -from .utils import * - -def make_sample_resource(identifier, long_name): - return { - 'source_name': 'hello', - 'source_copyright': [ - sample_file_ref('report.spdx'), - sample_file_ref('LICENSES/CC0-1.0.txt') - ], - 'type': 'resource', - 'identifier': identifier, - 'long_name': long_name, - 'uuid': 'a6754dcb-58d8-4b7a-a245-24fd7ad4cd68', - 'version': [2021, 11, 10], - 'revision': 1, - 'description': 'greets an apple', - 'dependencies': [{'identifier': 'hello-message'}], - 'scripts': [ - sample_file_ref('hello.js'), - sample_file_ref('bye.js') - ] - } - -def make_sample_mapping(identifier, long_name): - return { - 'source_name': 'example-org-fixes-new', - 'source_copyright': [ - sample_file_ref('report.spdx'), - sample_file_ref('LICENSES/CC0-1.0.txt') - ], - 'type': 'mapping', - 'identifier': identifier, - 'long_name': long_name, - 'uuid': '54d23bba-472e-42f5-9194-eaa24c0e3ee7', - 'version': [2022, 5, 10], - 'description': 'suckless something something', - 'payloads': { - 'https://example.org/a/*': { - 'identifier': 'some-KISS-resource' - }, - 'https://example.org/t/*': { - 'identifier': 'another-KISS-resource' - } - } - } - -def make_item(item_type, *args): - return make_sample_resource(*args) if item_type == 'resource' \ - else make_sample_mapping(*args) - -@pytest.mark.ext_data({ - 'background_script': broker_js, - 'extra_html': ExtraHTML('html/item_list.html', {}), - 'navigate_to': 'html/item_list.html' -}) -@pytest.mark.usefixtures('webextension') -@pytest.mark.parametrize('item_type', ['resource', 'mapping']) -def test_item_list_ordering(driver, execute_in_page, item_type): - """ - A test case of items list proper ordering. - """ - execute_in_page(load_script('html/item_list.js')) - - # Choose sample long names so as to test automatic sorting of items. - long_names = ['sample', 'sample it', 'Sample it', 'SAMPLE IT', - 'test', 'test it', 'Test it', 'TEST IT'] - # Let's operate on a reverse-sorted copy - long_names_reversed = [*long_names] - long_names_reversed.reverse() - - items = [make_item(item_type, f'it_{hex(2 * i + copy)[-1]}', name) - for i, name in enumerate(long_names_reversed) - for copy in (1, 0)] - # When adding/updating items this item will be updated at the end and this - # last update will be used to verify that a set of opertions completed. - extra_item = make_item(item_type, 'extraitem', 'extra item') - - # After this reversal items are sorted in the exact order they are expected - # to appear in the HTML list. - items.reverse() - - sample_data = { - 'resource': {}, - 'mapping': {}, - 'file': { - 'sha256': sample_files_by_sha256 - } - } - - indexes_added = set() - for iteration, to_include in enumerate([ - set([i for i in range(len(items)) if is_prime(i)]), - set([i for i in range(len(items)) - if not is_prime(i) and i & 1]), - set([i for i in range(len(items)) if i % 3 == 0]), - set([i for i in range(len(items)) - if i % 3 and not i & 1 and not is_prime(i)]), - set(range(len(items))) - ]): - # On the last iteration, re-add ALL items but with changed names. - if len(to_include) == len(items): - for it in items: - it['long_name'] = f'somewhat renamed {it["long_name"]}' - - items_to_inclue = [items[i] for i in sorted(to_include)] - sample_data[item_type] = sample_data_dict(items_to_inclue) - execute_in_page('returnval(haketilodb.save_items(arguments[0]));', - sample_data) - - extra_item['long_name'] = f'{iteration} {extra_item["long_name"]}' - sample_data[item_type] = sample_data_dict([extra_item]) - execute_in_page('returnval(haketilodb.save_items(arguments[0]));', - sample_data) - - if iteration == 0: - execute_in_page( - f''' - let list_ctx; - async function create_list() {{ - list_ctx = await {item_type}_list(); - document.body.append(list_ctx.main_div); - }} - returnval(create_list()); - ''') - - def lis_ready(driver): - return extra_item['long_name'] == execute_in_page( - 'returnval(list_ctx.ul.firstElementChild.textContent);' - ) - - indexes_added.update(to_include) - WebDriverWait(driver, 10).until(lis_ready) - - li_texts = execute_in_page( - ''' - var lis = [...list_ctx.ul.children].slice(1); - returnval(lis.map(li => li.textContent)); - ''') - assert li_texts == [items[i]['long_name'] for i in indexes_added] - - preview_texts = execute_in_page( - '''{ - const get_texts = - li => [li.click(), list_ctx.preview_container.textContent][1]; - returnval(lis.map(get_texts)); - }''') - - for i, text in zip(sorted(indexes_added), preview_texts): - assert items[i]['identifier'] in text - assert items[i]['long_name'] in text - -@pytest.mark.ext_data({ - 'background_script': broker_js, - 'extra_html': ExtraHTML('html/item_list.html', {}), - 'navigate_to': 'html/item_list.html' -}) -@pytest.mark.usefixtures('webextension') -@pytest.mark.parametrize('item_type', ['resource', 'mapping']) -def test_item_list_displaying(driver, execute_in_page, item_type): - """ - A test case of items list interaction with preview and dialog. - """ - execute_in_page(load_script('html/item_list.js')) - - items = [make_item(item_type, f'item{i}', f'Item {i}') for i in range(3)] - - sample_data = { - 'resource': {}, - 'mapping': {}, - 'file': { - 'sha256': sample_files_by_sha256 - } - } - sample_data[item_type] = sample_data_dict(items) - - preview_container, dialog_container, ul = execute_in_page( - f''' - let list_ctx, sample_data = arguments[0]; - async function create_list() {{ - await haketilodb.save_items(sample_data); - list_ctx = await {item_type}_list(); - document.body.append(list_ctx.main_div); - return [list_ctx.preview_container, list_ctx.dialog_container, - list_ctx.ul]; - }} - returnval(create_list()); - ''', - sample_data) - - assert not preview_container.is_displayed() - - # Check that preview is displayed correctly. - for i in range(3): - execute_in_page('list_ctx.ul.children[arguments[0]].click();', i) - assert preview_container.is_displayed() - text = preview_container.text - assert f'item{i}' in text - assert f'Item {i}' in text - - # Check that item removal confirmation dialog is displayed correctly. - execute_in_page('list_ctx.remove_but.click();') - WebDriverWait(driver, 10).until(lambda _: dialog_container.is_displayed()) - assert 'list_disabled' in ul.get_attribute('class') - assert not preview_container.is_displayed() - msg = execute_in_page('returnval(list_ctx.dialog_ctx.msg.textContent);') - assert msg == "Are you sure you want to delete 'item2'?" - - # Check that previewing other item is impossible while dialog is open. - execute_in_page('list_ctx.ul.children[0].click();') - assert dialog_container.is_displayed() - assert 'list_disabled' in ul.get_attribute('class') - assert not preview_container.is_displayed() - - # Check that queuing multiple removal confirmation dialogs is impossible. - execute_in_page('list_ctx.remove_but.click();') - - # Check that answering "No" causes the item not to be removed and unhides - # item preview. - execute_in_page('list_ctx.dialog_ctx.no_but.click();') - WebDriverWait(driver, 10).until(lambda _: preview_container.is_displayed()) - assert not dialog_container.is_displayed() - assert 'list_disabled' not in ul.get_attribute('class') - assert execute_in_page('returnval(list_ctx.ul.children.length);') == 3 - - # Check that item removal works properly. - def remove_current_item(): - execute_in_page('list_ctx.remove_but.click();') - WebDriverWait(driver, 10)\ - .until(lambda _: dialog_container.is_displayed()) - execute_in_page('list_ctx.dialog_ctx.yes_but.click();') - - remove_current_item() - - def item_deleted(driver): - return execute_in_page('returnval(list_ctx.ul.children.length);') == 2 - WebDriverWait(driver, 10).until(item_deleted) - assert not dialog_container.is_displayed() - assert not preview_container.is_displayed() - assert 'list_disabled' not in ul.get_attribute('class') - - execute_in_page('list_ctx.ul.children[1].click();') - - # Check that item removal failure causes the right error dialog to appear. - execute_in_page('haketilodb.finalize_transaction = () => {throw "sth";};') - remove_current_item() - WebDriverWait(driver, 10).until(lambda _: dialog_container.is_displayed()) - msg = execute_in_page('returnval(list_ctx.dialog_ctx.msg.textContent);') - assert msg == "Couldn't remove 'item1' :(" - - # Destroy item list. - assert True == execute_in_page( - ''' - const main_div = list_ctx.main_div; - destroy_list(list_ctx); - returnval(main_div.parentElement === null); - ''') diff --git a/test/unit/test_item_preview.py b/test/unit/test_item_preview.py deleted file mode 100644 index fe9a98e..0000000 --- a/test/unit/test_item_preview.py +++ /dev/null @@ -1,208 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -""" -Haketilo unit tests - displaying resources and mappings details -""" - -# This file is part of Haketilo -# -# Copyright (C) 2022, Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -import pytest -from selenium.webdriver.support.ui import WebDriverWait -from selenium.common.exceptions import NoSuchWindowException - -from ..extension_crafting import ExtraHTML -from ..script_loader import load_script -from .utils import * - -@pytest.mark.ext_data({ - 'extra_html': ExtraHTML('html/item_preview.html', {}), - 'navigate_to': 'html/item_preview.html' -}) -@pytest.mark.usefixtures('webextension') -def test_resource_preview(driver, execute_in_page): - """ - A test case of the resource preview display function. - """ - execute_in_page(load_script('html/item_preview.js')) - - sample_resource = make_sample_resource() - - preview_div = execute_in_page( - ''' - let preview_object = resource_preview(arguments[0]); - document.body.append(preview_object.main_div); - returnval(preview_object.main_div); - ''', - sample_resource) - text = preview_div.text - - assert '...' not in text - - for string in [ - *filter(lambda v: type(v) is str, sample_resource.values()), - *[rr['identifier'] for rr in sample_resource['dependencies']], - *[c['file'] for k in ('source_copyright', 'scripts') - for c in sample_resource[k]], - item_version_string(sample_resource, True) - ]: - assert string in text - - sample_resource['identifier'] = 'hellopear' - sample_resource['long_name'] = 'Hello Pear' - sample_resource['description'] = 'greets a pear' - sample_resource['dependencies'] = [{'identifier': 'hello-msg'}] - for key in ('scripts', 'source_copyright'): - for file_ref in sample_resource[key]: - file_ref['file'] = file_ref['file'].replace('.', '_') - - preview_div = execute_in_page( - ''' - returnval(resource_preview(arguments[0], preview_object).main_div); - ''', - sample_resource) - text = preview_div.text - - for string in ['...', 'pple', 'hello-message', 'report.spdx', - 'LICENSES/CC0-1.0.txt', 'hello.js', 'bye.js']: - assert string not in text - - for string in ['hellopear', 'Hello Pear', 'hello-msg', 'greets a pear', - 'report_spdx', 'LICENSES/CC0-1_0_txt', 'hello_js', 'bye_js']: - assert string in text - -@pytest.mark.ext_data({ - 'extra_html': ExtraHTML('html/item_preview.html', {}), - 'navigate_to': 'html/item_preview.html' -}) -@pytest.mark.usefixtures('webextension') -def test_mapping_preview(driver, execute_in_page): - """ - A test case of the mapping preview display function. - """ - execute_in_page(load_script('html/item_preview.js')) - - sample_mapping = make_sample_mapping() - - preview_div = execute_in_page( - ''' - let preview_object = mapping_preview(arguments[0]); - document.body.append(preview_object.main_div); - returnval(preview_object.main_div); - ''', - sample_mapping) - text = preview_div.text - - assert '...' not in text - - for string in [ - *filter(lambda v: type(v) is str, sample_mapping.values()), - *[p['identifier'] for p in sample_mapping['payloads'].values()], - *[c['file'] for c in sample_mapping['source_copyright']], - item_version_string(sample_mapping) - ]: - assert string in text - - sample_mapping['identifier'] = 'example-org-bloated' - sample_mapping['long_name'] = 'Example.org Bloated', - sample_mapping['payloads'] = dict( - [(pat.replace('.org', '.com'), res_id) - for pat, res_id in sample_mapping['payloads'].items()] - ) - for file_ref in sample_mapping['source_copyright']: - file_ref['file'] = file_ref['file'].replace('.', '_') - - preview_div = execute_in_page( - ''' - returnval(mapping_preview(arguments[0], preview_object).main_div); - ''', - sample_mapping) - text = preview_div.text - - for string in ['...', 'inimal', 'example.org', 'report.spdx', - 'LICENSES/CC0-1.0.txt']: - assert string not in text - - for string in ['example-org-bloated', 'Example.org Bloated', 'example.com', - 'report_spdx', 'LICENSES/CC0-1_0_txt']: - assert string in text - -@pytest.mark.ext_data({ - 'background_script': broker_js, - 'extra_html': [ - ExtraHTML('html/item_preview.html', {}), - ExtraHTML('html/file_preview.html', {}, wrap_into_htmldoc=False) - ], - 'navigate_to': 'html/item_preview.html' -}) -@pytest.mark.usefixtures('webextension') -def test_file_preview_link(driver, execute_in_page): - """ - A test case of links created by preview functions that allow a - referenced file to be previewed. - """ - execute_in_page(load_script('html/item_preview.js')) - - sample_data = make_complete_sample_data() - sample_data['mapping'] = {} - execute_in_page('returnval(haketilodb.save_items(arguments[0]));', - sample_data) - - # Cause the "link" to `bye.js` to be invalid. - sample_resource = make_sample_resource() - sample_resource['scripts'][1]['sha256'] = 'dummy nonexistent hash' - - execute_in_page( - ''' - let resource_preview_object = resource_preview(arguments[0], undefined); - document.body.append(resource_preview_object.main_div); - ''', - sample_resource) - - window0 = driver.window_handles[0] - driver.find_element_by_link_text('hello.js').click() - - def blob_url_navigated(driver): - if len(driver.window_handles) < 2: - return - window1 = [wh for wh in driver.window_handles if wh != window0][0] - driver.switch_to.window(window1) - try: - return driver.current_url.startswith('blob') - except NoSuchWindowException: - pass - - WebDriverWait(driver, 10).until(blob_url_navigated) - - assert sample_files['hello.js']['contents'].strip() \ - in driver.find_element_by_tag_name("pre").text - - driver.close() - driver.switch_to.window(window0) - - driver.find_element_by_link_text('bye.js').click() - - def get_error_span(driver): - if len(driver.window_handles) < 2: - return - window1 = [wh for wh in driver.window_handles if wh != window0][0] - driver.switch_to.window(window1) - try: - return driver.find_element_by_id('error_msg') - except NoSuchWindowException: - pass - - error_span = WebDriverWait(driver, 10).until(get_error_span) - assert error_span.is_displayed() - assert "Couldn't find file in Haketilo's internal database :(" \ - in error_span.text diff --git a/test/unit/test_patterns.py b/test/unit/test_patterns.py deleted file mode 100644 index f2eeaf8..0000000 --- a/test/unit/test_patterns.py +++ /dev/null @@ -1,152 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -""" -Haketilo unit tests - URL patterns -""" - -# This file is part of Haketilo -# -# Copyright (C) 2021, Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -import pytest - -from ..script_loader import load_script - -@pytest.mark.get_page('https://gotmyowndoma.in') -def test_regexes(execute_in_page): - """ - patterns.js contains regexes used for URL parsing. - Verify they work properly. - """ - execute_in_page(load_script('common/patterns.js')) - - valid_url = 'https://example.com/a/b?ver=1.2.3#heading2' - valid_url_rest = 'example.com/a/b?ver=1.2.3#heading2' - - # Test matching of URL protocol. - match = execute_in_page('returnval(proto_regex.exec(arguments[0]));', - valid_url) - assert match - assert match[1] == 'https' - assert match[2] == valid_url_rest - - match = execute_in_page('returnval(proto_regex.exec(arguments[0]));', - '://bad-url.missing/protocol') - assert match is None - - # Test matching of http(s) URLs. - match = execute_in_page('returnval(http_regex.exec(arguments[0]));', - valid_url_rest) - assert match - assert match[1] == 'example.com' - assert match[2] == '/a/b' - assert match[3] == '?ver=1.2.3' - - match = execute_in_page('returnval(http_regex.exec(arguments[0]));', - 'another.example.com') - assert match - assert match[1] == 'another.example.com' - assert match[2] == '' - assert match[3] == '' - - match = execute_in_page('returnval(http_regex.exec(arguments[0]));', - '/bad/http/example') - assert match == None - - # Test matching of file URLs. - match = execute_in_page('returnval(file_regex.exec(arguments[0]));', - '/good/file/example') - assert match - assert match[1] == '/good/file/example' - - # Test matching of ftp URLs. - match = execute_in_page('returnval(ftp_regex.exec(arguments[0]));', - 'example.com/a/b#heading2') - assert match - assert match[1] is None - assert match[2] == 'example.com' - assert match[3] == '/a/b' - - match = execute_in_page('returnval(ftp_regex.exec(arguments[0]));', - 'some_user@localhost') - assert match - assert match[1] == 'some_user@' - assert match[2] == 'localhost' - assert match[3] == '' - - match = execute_in_page('returnval(ftp_regex.exec(arguments[0]));', - '@bad.url/') - assert match is None - -@pytest.mark.get_page('https://gotmyowndoma.in') -def test_deconstruct_url(execute_in_page): - """ - patterns.js contains deconstruct_url() function that handles URL parsing. - Verify it works properly. - """ - execute_in_page(load_script('common/patterns.js')) - - deco = execute_in_page('returnval(deconstruct_url(arguments[0]));', - 'https://eXaMpLe.com/a/b?ver=1.2.3#heading2') - assert deco - assert deco['trailing_slash'] == False - assert deco['proto'] == 'https' - assert deco['domain'] == ['example', 'com'] - assert deco['path'] == ['a', 'b'] - - deco = execute_in_page('returnval(deconstruct_url(arguments[0]));', - 'http://**.example.com/') - assert deco - assert deco['trailing_slash'] == True - assert deco['proto'] == 'http' - assert deco['domain'] == ['**', 'example', 'com'] - assert deco['path'] == [] - - deco = execute_in_page('returnval(deconstruct_url(arguments[0]));', - 'ftp://user@ftp.example.com/all///passwords.txt/') - assert deco - assert deco['trailing_slash'] == True - assert deco['proto'] == 'ftp' - assert deco['domain'] == ['ftp', 'example', 'com'] - assert deco['path'] == ['all', 'passwords.txt'] - - deco = execute_in_page('returnval(deconstruct_url(arguments[0]));', - 'ftp://mirror.edu.pl.eu.org') - assert deco - assert deco['trailing_slash'] == False - assert deco['proto'] == 'ftp' - assert deco['domain'] == ['mirror', 'edu', 'pl', 'eu', 'org'] - assert deco['path'] == [] - - deco = execute_in_page('returnval(deconstruct_url(arguments[0]));', - 'file:///mnt/parabola_chroot///etc/passwd') - assert deco - assert deco['trailing_slash'] == False - assert deco['proto'] == 'file' - assert deco['path'] == ['mnt', 'parabola_chroot', 'etc', 'passwd'] - assert 'domain' not in deco - - for bad_url in [ - '://bad-url.missing/protocol', - 'http:/example.com/a/b', - 'unknown://example.com/a/b', - 'idontfancypineapple', - 'ftp://@example.org/', - 'https:///some/path/', - 'file://non-absolute/path' - ]: - with pytest.raises(Exception, match=r'Error in injected script'): - deco = execute_in_page('returnval(deconstruct_url(arguments[0]));', - bad_url) - - # at some point we might also consider testing url deconstruction with - # length limits... diff --git a/test/unit/test_patterns_query_manager.py b/test/unit/test_patterns_query_manager.py deleted file mode 100644 index 9fbc438..0000000 --- a/test/unit/test_patterns_query_manager.py +++ /dev/null @@ -1,307 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -""" -Haketilo unit tests - building pattern tree and putting it in a content script -""" - -# This file is part of Haketilo -# -# Copyright (C) 2021,2022 Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -import pytest -import json -from selenium.webdriver.support.ui import WebDriverWait -from selenium.common.exceptions import TimeoutException - -from ..script_loader import load_script - -def simple_sample_mapping(patterns, fruit): - if type(patterns) is not list: - patterns = [patterns] - payloads = dict([(p, {'identifier': f'{fruit}-{p}'}) for p in patterns]) - return { - 'source_copyright': [], - 'type': 'mapping', - 'identifier': f'inject-{fruit}', - 'payloads': payloads - } - -def get_content_script_values(driver, content_script): - """ - Allow easy extraction of 'this.something = ...' values from generated - content script and verify the content script is syntactically correct. - """ - return driver.execute_script( - ''' - function value_holder() { - %s; - return this; - } - return value_holder.call({}); - ''' % content_script) - -# Fields that are not relevant for testing are omitted from these mapping -# definitions. -sample_mappings = [simple_sample_mapping(pats, fruit) for pats, fruit in [ - (['https://gotmyowndoma.in/index.html', - 'http://gotmyowndoma.in/index.html'], 'banana'), - (['https://***.gotmyowndoma.in/index.html', - 'https://**.gotmyowndoma.in/index.html', - 'https://*.gotmyowndoma.in/index.html', - 'https://gotmyowndoma.in/index.html'], 'orange'), - ('https://gotmyowndoma.in/index.html/***', 'grape'), - ('http://gotmyowndoma.in/index.html/***', 'melon'), - ('https://gotmyowndoma.in/index.html', 'peach'), - ('https://gotmyowndoma.in/*', 'pear'), - ('https://gotmyowndoma.in/**', 'raspberry'), - ('https://gotmyowndoma.in/***', 'strawberry'), - ('https://***.gotmyowndoma.in/index.html', 'apple'), - ('https://***.gotmyowndoma.in/*', 'avocado'), - ('https://***.gotmyowndoma.in/**', 'papaya'), - ('https://***.gotmyowndoma.in/***', 'kiwi') -]] - -sample_blocking = [f'http{s}://{dw}gotmyown%sdoma.in{i}{pw}' - for dw in ('', '***.', '**.', '*.') - for i in ('/index.html', '') - for pw in ('', '/', '/*') - for s in ('', 's')] -sample_blocking = [{'pattern': pattern % (i if i > 1 else ''), - 'allow': bool(i & 1)} - for i, pattern in enumerate(sample_blocking)] - -# Even though patterns_query_manager.js is normally meant to run from background -# page, some tests can be as well performed running it from a normal page. -@pytest.mark.get_page('https://gotmyowndoma.in') -def test_pqm_tree_building(driver, execute_in_page): - """ - patterns_query_manager.js tracks Haketilo's internal database and builds a - constantly-updated pattern tree based on its contents. Mock the database and - verify tree building works properly. - """ - execute_in_page(load_script('background/patterns_query_manager.js')) - # Mock IndexedDB and build patterns tree. - execute_in_page( - ''' - const [initial_mappings, initial_blocking] = arguments.slice(0, 2); - let mappingchange, blockingchange, settingchange; - - haketilodb.track.mapping = function (cb) { - mappingchange = cb; - - return [{}, initial_mappings]; - } - haketilodb.track.blocking = function (cb) { - blockingchange = cb; - - return [{}, initial_blocking]; - } - haketilodb.track.setting = function (cb) { - settingchange = cb; - - return [{}, [{name: "default_allow", value: true}]]; - } - - let last_script; - let unregister_called = 0; - async function register_mock(injection) - { - await new Promise(resolve => setTimeout(resolve, 1)); - last_script = injection.js[0].code; - return {unregister: () => unregister_called++}; - } - browser = {contentScripts: {register: register_mock}}; - - returnval(start("abracadabra")); - ''', - sample_mappings[0:2], sample_blocking[0:2]) - - found, tree, content_script, deregistrations = execute_in_page( - ''' - returnval([pqt.search(tree, arguments[0]).next().value, - tree, last_script, unregister_called]); - ''', - 'https://gotmyowndoma.in/index.html') - best_pattern = 'https://gotmyowndoma.in/index.html' - assert found == \ - dict([('~allow', 1), - *[(f'inject-{fruit}', {'identifier': f'{fruit}-{best_pattern}'}) - for fruit in ('banana', 'orange')]]) - cs_values = get_content_script_values(driver, content_script) - assert cs_values['haketilo_secret'] == 'abracadabra' - assert cs_values['haketilo_pattern_tree'] == tree - assert cs_values['haketilo_default_allow'] == True - assert deregistrations == 0 - - def condition_all_added(driver): - last_script = execute_in_page('returnval(last_script);') - cs_values = get_content_script_values(driver, last_script) - nums = [i for i in range(len(sample_blocking)) if i > 1] - return (cs_values['haketilo_default_allow'] == False and - all([('gotmyown%sdoma' % i) in last_script for i in nums]) and - all([m['identifier'] in last_script for m in sample_mappings])) - - execute_in_page( - '''{ - const new_setting_val = {name: "default_allow", value: false}; - settingchange({key: "default_allow", new_val: new_setting_val}); - for (const mapping of arguments[0]) - mappingchange({key: mapping.identifier, new_val: mapping}); - for (const blocking of arguments[1]) - blockingchange({key: blocking.pattern, new_val: blocking}); - }''', - sample_mappings[2:], sample_blocking[2:]) - WebDriverWait(driver, 10).until(condition_all_added) - - odd_mappings = \ - [m['identifier'] for i, m in enumerate(sample_mappings) if i & 1] - odd_blocking = \ - [b['pattern'] for i, b in enumerate(sample_blocking) if i & 1] - even_mappings = \ - [m['identifier'] for i, m in enumerate(sample_mappings) if 1 - i & 1] - even_blocking = \ - [b['pattern'] for i, b in enumerate(sample_blocking) if 1 - i & 1] - - def condition_odd_removed(driver): - last_script = execute_in_page('returnval(last_script);') - nums = [i for i in range(len(sample_blocking)) if i > 1 and 1 - i & 1] - return (all([id not in last_script for id in odd_mappings]) and - all([id in last_script for id in even_mappings]) and - all([p not in last_script for p in odd_blocking[1:]]) and - all([('gotmyown%sdoma' % i) in last_script for i in nums])) - - def condition_all_removed(driver): - content_script = execute_in_page('returnval(last_script);') - cs_values = get_content_script_values(driver, content_script) - return cs_values['haketilo_pattern_tree'] == {} - - execute_in_page( - ''' - arguments[0].forEach(identifier => mappingchange({key: identifier})); - arguments[1].forEach(pattern => blockingchange({key: pattern})); - ''', - odd_mappings, odd_blocking) - - WebDriverWait(driver, 10).until(condition_odd_removed) - - execute_in_page( - ''' - arguments[0].forEach(identifier => mappingchange({key: identifier})); - arguments[1].forEach(pattern => blockingchange({key: pattern})); - ''', - even_mappings, even_blocking) - - WebDriverWait(driver, 10).until(condition_all_removed) - - def condition_default_allowed_again(driver): - content_script = execute_in_page('returnval(last_script);') - cs_values = get_content_script_values(driver, content_script) - return cs_values['haketilo_default_allow'] == True - - execute_in_page( - '''{ - const new_setting_val = {name: "default_allow", value: true}; - settingchange({key: "default_allow", new_val: new_setting_val}); - }''') - - WebDriverWait(driver, 10).until(condition_default_allowed_again) - -content_js = ''' -let already_run = false; -this.haketilo_content_script_main = function() { - if (already_run) - return; - already_run = true; - document.documentElement.innerHTML = "
"; - document.getElementById("tree-json").innerText = - JSON.stringify(this.haketilo_pattern_tree); -} -if (this.haketilo_pattern_tree !== undefined) - this.haketilo_content_script_main(); -''' - -def background_js(): - pqm_js = load_script('background/patterns_query_manager.js', - "#IMPORT background/broadcast_broker.js") - return pqm_js + '; broadcast_broker.start(); start();' - -@pytest.mark.ext_data({ - 'content_script': content_js, - 'background_script': background_js -}) -@pytest.mark.usefixtures('webextension') -def test_pqm_script_injection(driver, execute_in_page): - # Let's open a normal page in a second window. Window 0 will be used to make - # changes to IndexedDB and window 1 to test the working of content scripts. - driver.execute_script('window.open("about:blank", "_blank");') - WebDriverWait(driver, 10).until(lambda d: len(d.window_handles) == 2) - windows = [*driver.window_handles] - - def get_tree_json(driver): - return driver.execute_script( - ''' - return (document.getElementById("tree-json") || {}).innerText; - ''') - - def run_content_script(): - driver.switch_to.window(windows[1]) - driver.get('https://gotmyowndoma.in/index.html') - windows[1] = driver.current_window_handle - try: - return WebDriverWait(driver, 10).until(get_tree_json) - except TimeoutException: - pass - - for attempt in range(2): - json_txt = run_content_script() - if json_txt and json.loads(json_txt) == {}: - break; - assert attempt != 1 - - driver.switch_to.window(windows[0]) - execute_in_page(load_script('common/indexeddb.js')) - - sample_data = { - 'mapping': dict([(sm['identifier'], {'1.0': sm}) - for sm in sample_mappings]), - 'resource': {}, - 'file': {} - } - execute_in_page('returnval(save_items(arguments[0]));', sample_data) - - for attempt in range(2): - tree_json = run_content_script() or '{}' - json.loads(tree_json) - if all([m['identifier'] in tree_json for m in sample_mappings]): - break - assert attempt != 1 - - driver.switch_to.window(windows[0]) - execute_in_page( - '''{ - const identifiers = arguments[0]; - async function remove_items() - { - const ctx = await start_items_transaction(["mapping"], {}); - for (const id of identifiers) - await remove_mapping(id, ctx); - await finalize_transaction(ctx); - } - returnval(remove_items()); - }''', - [sm['identifier'] for sm in sample_mappings]) - - for attempt in range(2): - json_txt = run_content_script() - if json_txt and json.loads(json_txt) == {}: - break; - assert attempt != 1 diff --git a/test/unit/test_patterns_query_tree.py b/test/unit/test_patterns_query_tree.py deleted file mode 100644 index 80bf554..0000000 --- a/test/unit/test_patterns_query_tree.py +++ /dev/null @@ -1,474 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -""" -Haketilo unit tests - URL patterns -""" - -# This file is part of Haketilo -# -# Copyright (C) 2021, Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -import pytest - -from ..script_loader import load_script - -@pytest.mark.get_page('https://gotmyowndoma.in') -def test_modify_branch(execute_in_page): - """ - patterns_query_tree.js contains Pattern Tree data structure that allows - arrays of string labels to be mapped to items. - Verify operations modifying a single branch of such tree work properly. - """ - execute_in_page(load_script('common/patterns_query_tree.js')) - execute_in_page( - ''' - let items_added; - let items_removed; - - function _item_adder(item, array) - { - items_added++; - return [...(array || []), item]; - } - - function item_adder(item) - { - items_added = 0; - return array => _item_adder(item, array); - } - - function _item_remover(array) - { - if (array !== null) { - items_removed++; - array.pop(); - } - return (array && array.length > 0) ? array : null; - } - - function item_remover() - { - items_removed = 0; - return _item_remover; - }''') - - # Let's construct some tree branch while checking that each addition gives - # the right result. - branch = execute_in_page( - '''{ - const branch = empty_node(); - modify_sequence(branch, ['com', 'example'], item_adder('some_item')); - returnval(branch); - }''') - assert branch == { - 'literal_match': None, - 'wildcard_matches': [None, None, None], - 'children': { - 'com': { - 'literal_match': None, - 'wildcard_matches': [None, None, None], - 'children': { - 'example': { - 'literal_match': ['some_item'], - 'wildcard_matches': [None, None, None], - 'children': { - } - } - } - } - } - } - - branch, items_added = execute_in_page( - '''{ - const branch = arguments[0]; - modify_sequence(branch, ['com', 'example'], item_adder('other_item')); - returnval([branch, items_added]); - }''', branch) - assert items_added == 1 - assert branch['children']['com']['children']['example']['literal_match'] \ - == ['some_item', 'other_item'] - - for i in range(3): - for expected_array in [['third_item'], ['third_item', '4th_item']]: - wildcard = '*' * (i + 1) - branch, items_added = execute_in_page( - '''{ - const branch = arguments[0]; - modify_sequence(branch, ['com', 'sample', arguments[1]], - item_adder(arguments[2])); - returnval([branch, items_added]); - }''', - branch, wildcard, expected_array[-1]) - assert items_added == 2 - sample = branch['children']['com']['children']['sample'] - assert sample['wildcard_matches'][i] == expected_array - assert sample['children'][wildcard]['literal_match'] \ - == expected_array - - branch, items_added = execute_in_page( - '''{ - const branch = arguments[0]; - modify_sequence(branch, ['org', 'koszko', '***', '123'], - item_adder('5th_item')); - returnval([branch, items_added]); - }''', - branch) - assert items_added == 1 - assert branch['children']['org']['children']['koszko']['children']['***']\ - ['children']['123']['literal_match'] == ['5th_item'] - - # Let's verify that removing a nonexistent element doesn't modify the tree. - branch2, items_removed = execute_in_page( - '''{ - const branch = arguments[0]; - modify_sequence(branch, ['com', 'not', 'registered', '*'], - item_remover()); - returnval([branch, items_removed]); - }''', - branch) - assert branch == branch2 - assert items_removed == 0 - - # Let's remove all elements in the tree branch while checking that each - # removal gives the right result. - branch, items_removed = execute_in_page( - '''{ - const branch = arguments[0]; - modify_sequence(branch, ['org', 'koszko', '***', '123'], - item_remover()); - returnval([branch, items_removed]); - }''', - branch) - assert items_removed == 1 - assert 'org' not in branch['children'] - - for i in range(3): - for expected_array in [['third_item'], None]: - wildcard = '*' * (i + 1) - branch, items_removed = execute_in_page( - '''{ - const branch = arguments[0]; - modify_sequence(branch, ['com', 'sample', arguments[1]], - item_remover()); - returnval([branch, items_removed]); - }''', - branch, wildcard) - assert items_removed == 2 - if i == 2 and expected_array == []: - break - sample = branch['children']['com']['children'].get('sample', {}) - assert sample.get('wildcard_matches', [None, None, None])[i] \ - == expected_array - assert sample.get('children', {}).get(wildcard, {})\ - .get('literal_match') == expected_array - - for i in range(2): - branch, items_removed = execute_in_page( - '''{ - const branch = arguments[0]; - modify_sequence(branch, ['com', 'example'], item_remover()); - returnval([branch, items_removed]); - }''', - branch) - assert items_removed == 1 - if i == 0: - assert branch['children']['com']['children']['example']\ - ['literal_match'] == ['some_item'] - else: - assert branch == { - 'literal_match': None, - 'wildcard_matches': [None, None, None], - 'children': { - } - } - -@pytest.mark.get_page('https://gotmyowndoma.in') -def test_search_branch(execute_in_page): - """ - patterns_query_tree.js contains Pattern Tree data structure that allows - arrays of string labels to be mapped to items. - Verify searching a single branch of such tree work properly. - """ - execute_in_page(load_script('common/patterns_query_tree.js')) - execute_in_page( - ''' - const item_adder = item => (array => [...(array || []), item]); - ''') - - # Let's construct some tree branch to test on. - execute_in_page( - ''' - var branch = empty_node(); - - for (const [item, sequence] of [ - ['(root)', []], - ['***', ['***']], - ['**', ['**']], - ['*', ['*']], - - ['a', ['a']], - ['A', ['a']], - ['b', ['b']], - - ['a/***', ['a', '***']], - ['A/***', ['a', '***']], - ['a/**', ['a', '**']], - ['A/**', ['a', '**']], - ['a/*', ['a', '*']], - ['A/*', ['a', '*']], - ['a/sth', ['a', 'sth']], - ['A/sth', ['a', 'sth']], - - ['b/***', ['b', '***']], - ['b/**', ['b', '**']], - ['b/*', ['b', '*']], - ['b/sth', ['b', 'sth']], - ]) - modify_sequence(branch, sequence, item_adder(item)); - ''') - - # Let's make the actual searches on our testing branch. - for sequence, expected in [ - ([], [{'(root)'}, {'***'}]), - (['a'], [{'a', 'A'}, {'a/***', 'A/***'}, {'*'}, {'***'}]), - (['b'], [{'b'}, {'b/***'}, {'*'}, {'***'}]), - (['c'], [ {'*'}, {'***'}]), - (['***'], [{'***'}, {'*'} ]), - (['**'], [{'**'}, {'*'}, {'***'}]), - (['**'], [{'**'}, {'*'}, {'***'}]), - (['*'], [{'*'}, {'***'}]), - - (['a', 'sth'], [{'a/sth', 'A/sth'}, {'a/*', 'A/*'}, {'a/***', 'A/***'}, {'**'}, {'***'}]), - (['b', 'sth'], [{'b/sth'}, {'b/*'}, {'b/***'}, {'**'}, {'***'}]), - (['a', 'hts'], [ {'a/*', 'A/*'}, {'a/***', 'A/***'}, {'**'}, {'***'}]), - (['b', 'hts'], [ {'b/*'}, {'b/***'}, {'**'}, {'***'}]), - (['a', '***'], [{'a/***', 'A/***'}, {'a/*', 'A/*'}, {'**'}, {'***'}]), - (['b', '***'], [{'b/***'}, {'b/*'}, {'**'}, {'***'}]), - (['a', '**'], [{'a/**', 'A/**'}, {'a/*', 'A/*'}, {'a/***', 'A/***'}, {'**'}, {'***'}]), - (['b', '**'], [{'b/**'}, {'b/*'}, {'b/***'}, {'**'}, {'***'}]), - (['a', '*'], [{'a/*', 'A/*'}, {'a/***', 'A/***'}, {'**'}, {'***'}]), - (['b', '*'], [{'b/*'}, {'b/***'}, {'**'}, {'***'}]), - - (['a', 'c', 'd'], [{'a/**', 'A/**'}, {'a/***', 'A/***'}, {'**'}, {'***'}]), - (['b', 'c', 'd'], [{'b/**'}, {'b/***'}, {'**'}, {'***'}]) - ]: - result = execute_in_page( - ''' - returnval([...search_sequence(branch, arguments[0])]); - ''', - sequence) - - try: - assert len(result) == len(expected) - - for expected_set, result_array in zip(expected, result): - assert len(expected_set) == len(result_array) - assert expected_set == set(result_array) - except Exception as e: - import sys - print('sequence:', sequence, '\nexpected:', expected, - '\nresult:', result, file=sys.stderr) - raise e from None - -@pytest.mark.get_page('https://gotmyowndoma.in') -def test_pattern_tree(execute_in_page): - """ - patterns_query_tree.js contains Pattern Tree data structure that allows - arrays of string labels to be mapped to items. - Verify operations on entire such tree work properly. - """ - execute_in_page(load_script('common/patterns_query_tree.js')) - - # Perform tests with all possible patterns for a simple URL. - url = 'https://example.com' - patterns = [ - 'https://example.com', - 'https://example.com/***', - 'https://***.example.com', - 'https://***.example.com/***' - ] - bad_patterns = [ - 'http://example.com', - 'https://a.example.com', - 'https://*.example.com', - 'https://**.example.com', - 'https://example.com/a', - 'https://example.com/*', - 'https://example.com/**', - ] - - expected = [{'key': p} for p in patterns] - - tree, result = execute_in_page( - '''{ - const tree = pattern_tree_make(); - for (const pattern of arguments[0].concat(arguments[1])) { - pattern_tree_register(tree, pattern, 'key', pattern); - pattern_tree_register(tree, pattern + '/', 'key', pattern + '/'); - } - returnval([tree, [...pattern_tree_search(tree, arguments[2])]]); - }''', - patterns, bad_patterns, url) - assert expected == result - - # Also verify that deregistering half of the good patterns works correctly. - patterns_removed = [pattern for i, pattern in enumerate(patterns) if i % 2] - patterns = [pattern for i, pattern in enumerate(patterns) if not (i % 2)] - expected = [{'key': p} for p in patterns] - tree, result = execute_in_page( - '''{ - const tree = arguments[0]; - for (const pattern of arguments[1]) { - pattern_tree_deregister(tree, pattern, 'key'); - pattern_tree_deregister(tree, pattern + '/', 'key'); - } - returnval([tree, [...pattern_tree_search(tree, arguments[2])]]); - }''', - tree, patterns_removed, url) - assert expected == result - - # Also verify that deregistering all the patterns works correctly. - tree = execute_in_page( - '''{ - const tree = arguments[0]; - for (const pattern of arguments[1].concat(arguments[2])) { - pattern_tree_deregister(tree, pattern, 'key'); - pattern_tree_deregister(tree, pattern + '/', 'key'); - } - returnval(tree); - }''', - tree, patterns, bad_patterns) - assert tree == {} - - # Perform tests with all possible patterns for a complex URL. - url = 'http://settings.query.example.com/google/tries/destroy/adblockers//' - patterns = [ - 'http://settings.query.example.com/google/tries/destroy/adblockers', - 'http://settings.query.example.com/google/tries/destroy/adblockers/***', - 'http://settings.query.example.com/google/tries/destroy/*', - 'http://settings.query.example.com/google/tries/destroy/***', - 'http://settings.query.example.com/google/tries/**', - 'http://settings.query.example.com/google/tries/***', - 'http://settings.query.example.com/google/**', - 'http://settings.query.example.com/google/***', - 'http://settings.query.example.com/**', - 'http://settings.query.example.com/***', - - 'http://***.settings.query.example.com/google/tries/destroy/adblockers', - 'http://***.settings.query.example.com/google/tries/destroy/adblockers/***', - 'http://***.settings.query.example.com/google/tries/destroy/*', - 'http://***.settings.query.example.com/google/tries/destroy/***', - 'http://***.settings.query.example.com/google/tries/**', - 'http://***.settings.query.example.com/google/tries/***', - 'http://***.settings.query.example.com/google/**', - 'http://***.settings.query.example.com/google/***', - 'http://***.settings.query.example.com/**', - 'http://***.settings.query.example.com/***', - 'http://*.query.example.com/google/tries/destroy/adblockers', - 'http://*.query.example.com/google/tries/destroy/adblockers/***', - 'http://*.query.example.com/google/tries/destroy/*', - 'http://*.query.example.com/google/tries/destroy/***', - 'http://*.query.example.com/google/tries/**', - 'http://*.query.example.com/google/tries/***', - 'http://*.query.example.com/google/**', - 'http://*.query.example.com/google/***', - 'http://*.query.example.com/**', - 'http://*.query.example.com/***', - 'http://***.query.example.com/google/tries/destroy/adblockers', - 'http://***.query.example.com/google/tries/destroy/adblockers/***', - 'http://***.query.example.com/google/tries/destroy/*', - 'http://***.query.example.com/google/tries/destroy/***', - 'http://***.query.example.com/google/tries/**', - 'http://***.query.example.com/google/tries/***', - 'http://***.query.example.com/google/**', - 'http://***.query.example.com/google/***', - 'http://***.query.example.com/**', - 'http://***.query.example.com/***', - 'http://**.example.com/google/tries/destroy/adblockers', - 'http://**.example.com/google/tries/destroy/adblockers/***', - 'http://**.example.com/google/tries/destroy/*', - 'http://**.example.com/google/tries/destroy/***', - 'http://**.example.com/google/tries/**', - 'http://**.example.com/google/tries/***', - 'http://**.example.com/google/**', - 'http://**.example.com/google/***', - 'http://**.example.com/**', - 'http://**.example.com/***', - 'http://***.example.com/google/tries/destroy/adblockers', - 'http://***.example.com/google/tries/destroy/adblockers/***', - 'http://***.example.com/google/tries/destroy/*', - 'http://***.example.com/google/tries/destroy/***', - 'http://***.example.com/google/tries/**', - 'http://***.example.com/google/tries/***', - 'http://***.example.com/google/**', - 'http://***.example.com/google/***', - 'http://***.example.com/**', - 'http://***.example.com/***' - ] - bad_patterns = [ - 'https://settings.query.example.com/google/tries/destroy/adblockers', - 'http://settings.query.example.com/google/tries/destroy/adblockers/a', - 'http://settings.query.example.com/google/tries/destroy/adblockers/*', - 'http://settings.query.example.com/google/tries/destroy/adblockers/**', - 'http://settings.query.example.com/google/tries/destroy/a', - 'http://settings.query.example.com/google/tries/destroy/**', - 'http://settings.query.example.com/google/tries/*', - 'http://a.settings.query.example.com/google/tries/destroy/adblockers', - 'http://*.settings.query.example.com/google/tries/destroy/adblockers', - 'http://**.settings.query.example.com/google/tries/destroy/adblockers', - 'http://a.query.example.com/google/tries/destroy/adblockers', - 'http://**.query.example.com/google/tries/destroy/adblockers', - 'http://*.example.com/google/tries/destroy/adblockers' - ] - - expected = [{'key': p + s} for p in patterns for s in ['/', '']] - - tree, result = execute_in_page( - '''{ - const tree = pattern_tree_make(); - for (const pattern of arguments[0].concat(arguments[1])) { - pattern_tree_register(tree, pattern, 'key', pattern); - pattern_tree_register(tree, pattern + '/', 'key', pattern + '/'); - } - returnval([tree, [...pattern_tree_search(tree, arguments[2])]]); - }''', - patterns, bad_patterns, url) - assert expected == result - - # Also verify that deregistering all patterns with trailing slash works - # correctly. - expected = [{'key': p} for p in patterns] - tree, result = execute_in_page( - '''{ - const tree = arguments[0]; - for (const pattern of arguments[1]) - pattern_tree_deregister(tree, pattern + '/', 'key'); - returnval([tree, [...pattern_tree_search(tree, arguments[2])]]); - }''', - tree, patterns, url) - assert expected == result - - # Also verify that deregistering all the patterns works correctly. - tree = execute_in_page( - '''{ - const tree = arguments[0]; - for (const pattern of arguments[1]) - pattern_tree_deregister(tree, pattern, 'key'); - for (const pattern of arguments[2]) { - pattern_tree_deregister(tree, pattern, 'key'); - pattern_tree_deregister(tree, pattern + '/', 'key'); - } - returnval(tree); - }''', - tree, patterns, bad_patterns) - assert tree == {} diff --git a/test/unit/test_payload_create.py b/test/unit/test_payload_create.py deleted file mode 100644 index 9689c37..0000000 --- a/test/unit/test_payload_create.py +++ /dev/null @@ -1,248 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -""" -Haketilo unit tests - using a form to create simple site payload -""" - -# This file is part of Haketilo -# -# Copyright (C) 2022 Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -import pytest -import re -from hashlib import sha256 - -from selenium.webdriver.support.ui import WebDriverWait - -from ..extension_crafting import ExtraHTML -from ..script_loader import load_script -from .utils import * - -uuidv4_re = re.compile( - r'^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$', - re.IGNORECASE -) - -sample_patterns = ''' -http://example.com/*** - -https://*.example.org/**''' - -sample_form_data = { - 'identifier': 'someid', - 'long_name': 'Some Name', - 'description': 'blah blah blah', - 'patterns': sample_patterns, - 'script': sample_files['hello.js']['contents'] -} - -def fill_form_with_sample_data(execute_in_page, sample_data_override={}, - form_ctx='form_ctx'): - form_data = sample_form_data.copy() - form_data.update(sample_data_override) - execute_in_page( - f''' - for (const [key, value] of Object.entries(arguments[0])) - {form_ctx}[key].value = value; - ''', - form_data) - return form_data - -cleared_form_inputs = { - 'identifier': '', - 'long_name': '', - 'description': '', - 'patterns': 'https://example.com/***', - 'script': 'console.log("Hello, World!");' -} -def assert_form_contents(execute_in_page, inputs=cleared_form_inputs): - inputs_keys = [*inputs.keys()] - values = execute_in_page( - 'returnval(arguments[0].map(i => form_ctx[i].value));', - inputs_keys - ) - for key, value in zip(inputs_keys, values): - assert inputs[key] == value - -@pytest.mark.ext_data({ - 'background_script': broker_js, - 'extra_html': ExtraHTML('html/payload_create.html', {}), - 'navigate_to': 'html/payload_create.html' -}) -@pytest.mark.usefixtures('webextension') -def test_payload_create_normal_usage(driver, execute_in_page): - """ - A test case of normal usage of simple payload creation form. - """ - execute_in_page(load_script('html/payload_create.js')) - - create_but, form_container, dialog_container = execute_in_page( - ''' - const form_ctx = payload_create_form(); - document.body.append(form_ctx.main_div); - returnval([form_ctx.create_but, form_ctx.form_container, - form_ctx.dialog_container]); - ''') - - assert patterns_doc_url == \ - driver.find_element_by_link_text('URL patterns').get_attribute('href') - - assert form_container.is_displayed() - assert not dialog_container.is_displayed() - - assert_form_contents(execute_in_page) - - form_data = fill_form_with_sample_data(execute_in_page) - - create_but.click() - - assert not form_container.is_displayed() - assert dialog_container.is_displayed() - - def success_reported(driver): - return 'Successfully saved payload' in dialog_container.text - - WebDriverWait(driver, 10).until(success_reported) - execute_in_page('form_ctx.dialog_ctx.ok_but.click();') - - assert form_container.is_displayed() - assert not dialog_container.is_displayed() - - def assert_db_contents(): - db_contents = get_db_contents(execute_in_page) - - assert uuidv4_re.match(db_contents['resource'][0]['uuid']) - - localid = f'local-{form_data["identifier"]}' - long_name = form_data['long_name'] or form_data['identifier'] - payloads = dict([(pat, {'identifier': localid}) - for pat in form_data['patterns'].split('\n') if pat]) - - assert db_contents['resource'] == [{ - 'source_name': localid, - 'source_copyright': [], - 'type': 'resource', - 'identifier': localid, - 'uuid': db_contents['resource'][0]['uuid'], - 'version': [1], - 'description': form_data['description'], - 'dependencies': [], - 'long_name': long_name, - 'scripts': [{ - 'file': 'payload.js', - 'sha256': sha256(form_data['script'].encode()).digest().hex() - }] - }] - - assert uuidv4_re.match(db_contents['mapping'][0]['uuid']) - assert db_contents['mapping'] == [{ - 'source_name': localid, - 'source_copyright': [], - 'type': 'mapping', - 'identifier': localid, - 'uuid': db_contents['mapping'][0]['uuid'], - 'version': [1], - 'description': form_data['description'], - 'long_name': long_name, - 'payloads': payloads - }] - - assert_db_contents() - - form_data = fill_form_with_sample_data(execute_in_page, { - 'long_name': '', - 'description': 'bam bam bam', - 'patterns': 'https://new.example.com/***', - 'script': sample_files['bye.js']['contents'] - }) - - create_but.click() - - for type in ('Resource', 'Mapping'): - def override_asked(driver): - return f"{type} 'local-someid' already exists. Override?" \ - in dialog_container.text - WebDriverWait(driver, 10).until(override_asked) - execute_in_page('form_ctx.dialog_ctx.yes_but.click();') - - assert_db_contents() - -@pytest.mark.ext_data({ - 'background_script': broker_js, - 'extra_html': ExtraHTML('html/payload_create.html', {}), - 'navigate_to': 'html/payload_create.html' -}) -@pytest.mark.usefixtures('webextension') -def test_payload_create_errors(driver, execute_in_page): - """ - A test case of various error the simple payload form might show. - """ - execute_in_page(load_script('html/payload_create.js')) - - create_but, dialog_container = execute_in_page( - ''' - const form_ctx = payload_create_form(); - document.body.append(form_ctx.main_div); - returnval([form_ctx.create_but, form_ctx.dialog_container]); - ''') - - for data_override, expected_msg in [ - ({'identifier': ''}, "The 'identifier' field is required!"), - ({'identifier': ':('}, 'Identifier may only contain '), - ({'script': ''}, "The 'script' field is required!"), - ({'patterns': ''}, "The 'URL patterns' field is required!"), - ({'patterns': ':d'}, "':d' is not a valid URL pattern. See here for more details."), - ({'patterns': '\n'.join(['http://example.com'] * 2)}, - "Pattern 'http://example.com' specified multiple times!") - ]: - # Attempt creating the payload - form_data = fill_form_with_sample_data(execute_in_page, data_override) - create_but.click() - # Verify the error message - assert expected_msg in dialog_container.text - - # Verify patterns documentation link. - if expected_msg == {'patterns': ':d'}: - doc_link_elem = driver.find_element_by_link_text('here') - assert doc_link.get_attribute('href') == patterns_doc_url - - # Verify the form was NOT cleared upon failed saving. - execute_in_page('form_ctx.dialog_ctx.ok_but.click();') - assert_form_contents(execute_in_page, form_data) - - # Add a sample item and attempt overriding it. - fill_form_with_sample_data(execute_in_page) - create_but.click() - WebDriverWait(driver, 10).until(lambda _: 'Succes' in dialog_container.text) - execute_in_page('form_ctx.dialog_ctx.ok_but.click();') - - # Verify that denying override leads to saving failure. - form_data = fill_form_with_sample_data(execute_in_page) - create_but.click() - WebDriverWait(driver, 10).until(lambda _: 'Overri' in dialog_container.text) - execute_in_page('form_ctx.dialog_ctx.no_but.click();') - assert 'Failed to save payload :(' in dialog_container.text - execute_in_page('form_ctx.dialog_ctx.ok_but.click();') - assert_form_contents(execute_in_page, form_data) - - # Verify that IndexedDB errors get caught and reported as saving failures. - execute_in_page('haketilodb.get = async () => {throw "someerror";}') - form_data = fill_form_with_sample_data(execute_in_page, {'identifier': 'o'}) - create_but.click() - WebDriverWait(driver, 10).until(lambda _: 'Failed' in dialog_container.text) - execute_in_page('form_ctx.dialog_ctx.ok_but.click();') - assert_form_contents(execute_in_page, form_data) - - # Verify that the loading message gets shown during IndexedDB operations. - execute_in_page('haketilodb.get = () => new Promise(cb => null);') - create_but.click() - assert 'Saving payload...' in dialog_container.text diff --git a/test/unit/test_policy_deciding.py b/test/unit/test_policy_deciding.py deleted file mode 100644 index 75b35ac..0000000 --- a/test/unit/test_policy_deciding.py +++ /dev/null @@ -1,135 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -""" -Haketilo unit tests - determining what to do on a given web page -""" - -# This file is part of Haketilo -# -# Copyright (C) 2021, Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -import re -from hashlib import sha256 -import pytest - -from ..script_loader import load_script - -csp_re = re.compile(r'^\S+\s+\S+;(?:\s+\S+\s+\S+;)*$') -rule_re = re.compile(r'^\s*(?P\S+)\s+(?P\S+)$') -def parse_csp(csp): - ''' - Parsing of CSP string into a dict. A simplified format of CSP is assumed. - ''' - assert csp_re.match(csp) - - result = {} - - for rule in csp.split(';')[:-1]: - match = rule_re.match(rule) - result[match.group('src_kind')] = match.group('allowed_origins') - - return result - -@pytest.mark.get_page('https://gotmyowndoma.in') -def test_decide_policy(execute_in_page): - """ - policy.js contains code that, using a Pattern Query Tree instance and a URL, - decides what Haketilo should do on a page opened at that URL, i.e. whether - it should block or allow script execution and whether it should inject its - own scripts and which ones. Test that the policy object gets constructed - properly. - """ - execute_in_page(load_script('common/policy.js')) - - policy = execute_in_page( - ''' - returnval(decide_policy(pqt.make(), "http://unkno.wn/", true, "abcd")); - ''') - assert policy['allow'] == True - for prop in ('mapping', 'payload', 'nonce', 'csp', 'error'): - assert prop not in policy - - policy = execute_in_page( - '''{ - const tree = pqt.make(); - pqt.register(tree, "http://kno.wn", "~allow", 1); - returnval(decide_policy(tree, "http://kno.wn/", false, "abcd")); - }''') - assert policy['allow'] == True - assert policy['mapping'] == '~allow' - for prop in ('payload', 'nonce', 'csp', 'error'): - assert prop not in policy - - policy = execute_in_page( - ''' - returnval(decide_policy(pqt.make(), "http://unkno.wn/", false, "abcd")); - ''' - ) - assert policy['allow'] == False - for prop in ('mapping', 'payload', 'nonce', 'error'): - assert prop not in policy - assert parse_csp(policy['csp']) == { - 'prefetch-src': "'none'", - 'script-src-attr': "'none'", - 'script-src': "'none'", - 'script-src-elem': "'none'" - } - - policy = execute_in_page( - '''{ - const tree = pqt.make(); - pqt.register(tree, "http://kno.wn", "~allow", 0); - returnval(decide_policy(tree, "http://kno.wn/", true, "abcd")); - }''') - assert policy['allow'] == False - assert policy['mapping'] == '~allow' - for prop in ('payload', 'nonce', 'error'): - assert prop not in policy - assert parse_csp(policy['csp']) == { - 'prefetch-src': "'none'", - 'script-src-attr': "'none'", - 'script-src': "'none'", - 'script-src-elem': "'none'" - } - - policy = execute_in_page( - '''{ - const tree = pqt.make(); - pqt.register(tree, "http://kno.wn", "m1", {identifier: "res1"}); - returnval(decide_policy(tree, "http://kno.wn/", true, "abcd")); - }''') - assert policy['allow'] == False - assert policy['mapping'] == 'm1' - assert policy['payload'] == {'identifier': 'res1'} - assert 'error' not in policy - assert policy['nonce'] == \ - sha256('m1:res1:http://kno.wn/:abcd'.encode()).digest().hex() - assert parse_csp(policy['csp']) == { - 'prefetch-src': f"'none'", - 'script-src-attr': f"'none'", - 'script-src': f"'nonce-{policy['nonce']}'", - 'script-src-elem': f"'nonce-{policy['nonce']}'" - } - - policy = execute_in_page( - 'returnval(decide_policy(pqt.make(), "", true, "abcd"));' - ) - assert policy['allow'] == False - assert policy['error'] == {'haketilo_error_type': 'deciding_policy'} - for prop in ('mapping', 'payload', 'nonce'): - assert prop not in policy - assert parse_csp(policy['csp']) == { - 'prefetch-src': "'none'", - 'script-src-attr': "'none'", - 'script-src': "'none'", - 'script-src-elem': "'none'" - } diff --git a/test/unit/test_policy_enforcing.py b/test/unit/test_policy_enforcing.py deleted file mode 100644 index 4b7c173..0000000 --- a/test/unit/test_policy_enforcing.py +++ /dev/null @@ -1,114 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -""" -Haketilo unit tests - enforcing script blocking policy from content script -""" - -# This file is part of Haketilo -# -# Copyright (C) 2022 Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the CC0 1.0 Universal License as published by -# the Creative Commons Corporation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# CC0 1.0 Universal License for more details. - -import pytest -import json -import urllib.parse -from selenium.webdriver.support.ui import WebDriverWait - -from ..script_loader import load_script -from .utils import are_scripts_allowed - -# For simplicity, we'll use one nonce in all test cases. -nonce = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' - -allow_policy = {'allow': True} -block_policy = { - 'allow': False, - 'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'none'; script-src-elem 'none'; frame-src http://* https://*;" -} -payload_policy = { - 'mapping': 'somemapping', - 'payload': {'identifier': 'someresource'}, - 'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'nonce-{nonce}'; script-src-elem 'nonce-{nonce}';" -} - -content_script = load_script('content/policy_enforcing.js') + ''';{ -const smuggled_what_to_do = /^[^#]*#?(.*)$/.exec(document.URL)[1]; -const what_to_do = smuggled_what_to_do === "" ? {policy: {allow: true}} : - JSON.parse(decodeURIComponent(smuggled_what_to_do)); - -if (what_to_do.csp_off) { - const orig_DOMParser = window.DOMParser; - window.DOMParser = function() { - const parser = new orig_DOMParser(); - this.parseFromString = () => parser.parseFromString('', 'text/html'); - } -} - -enforce_blocking(what_to_do.policy); -}''' - -def get(driver, page, what_to_do): - driver.get(page + '#' + urllib.parse.quote(json.dumps(what_to_do))) - driver.execute_script('window.before_reload = true; location.reload();') - done = lambda _: not driver.execute_script('return window.before_reload;') - WebDriverWait(driver, 10).until(done) - -@pytest.mark.ext_data({'content_script': content_script}) -@pytest.mark.usefixtures('webextension') -# Under Mozilla we use several mechanisms of script blocking. Some serve as -# fallbacks in case others break. CSP one of those mechanisms. Here we run the -# test once with CSP blocking on and once without it. This allows us to verify -# that the CSP-less blocking approaches by themselves also work. We don't do the -# reverse (CSP on and other mechanisms off) because CSP rules added through -# injection are not reliable enough - they do not always take effect -# immediately and there's nothing we can do to fix it. -@pytest.mark.parametrize('csp_off_setting', [{}, {'csp_off': True}]) -def test_policy_enforcing_html(driver, execute_in_page, csp_off_setting): - """ - A test case of sanitizing