From b75a5717a084c9e5a727c2e960f2b910abcb5ace Mon Sep 17 00:00:00 2001 From: Wojtek Kosior Date: Tue, 25 Jan 2022 09:37:34 +0100 Subject: add a repo querying HTML interface --- test/server.py | 7 +- test/unit/test_install.py | 30 ++-- test/unit/test_repo_query.py | 278 ++++++++++++++++++++++++++++++++++++ test/unit/test_repo_query_cacher.py | 42 +++++- test/unit/utils.py | 78 +++++----- test/world_wide_library.py | 62 ++++++-- 6 files changed, 426 insertions(+), 71 deletions(-) create mode 100644 test/unit/test_repo_query.py (limited to 'test') diff --git a/test/server.py b/test/server.py index 9cab5cc..0963b5b 100755 --- a/test/server.py +++ b/test/server.py @@ -31,6 +31,7 @@ wrapping the classes in proxy_core.py from pathlib import Path from urllib.parse import parse_qs from threading import Thread +import traceback from .proxy_core import ProxyRequestHandler, ThreadingHTTPServer from .misc_constants import * @@ -84,8 +85,10 @@ class RequestHijacker(ProxyRequestHandler): status_code, headers = 404, {'Content-Type': 'text/plain'} resp_body = b'Handler for this URL not found.' - except Exception as e: - status_code, headers, resp_body = 500, {'Content-Type': 'text/plain'}, b'Internal Error:\n' + repr(e).encode() + except Exception: + status_code = 500 + headers = {'Content-Type': 'text/plain'} + resp_body = b'Internal Error:\n' + traceback.format_exc().encode() headers['Content-Length'] = str(len(resp_body)) self.send_response(status_code) diff --git a/test/unit/test_install.py b/test/unit/test_install.py index cb8fe36..303d31a 100644 --- a/test/unit/test_install.py +++ b/test/unit/test_install.py @@ -25,31 +25,21 @@ from ..extension_crafting import ExtraHTML from ..script_loader import load_script from .utils import * -def content_script(): - script = load_script('content/repo_query_cacher.js') - return f'{script}; {tab_id_asker}; start();' - -def background_script(): - script = load_script('background/broadcast_broker.js', - '#IMPORT background/CORS_bypass_server.js') - return f'{script}; {tab_id_responder}; start(); CORS_bypass_server.start();' - def setup_view(driver, execute_in_page): - tab_id = run_content_script_in_new_window(driver, 'https://gotmyowndoma.in') + mock_cacher(execute_in_page) execute_in_page(load_script('html/install.js')) container_ids, containers_objects = execute_in_page( ''' const cb_calls = []; - const install_view = new InstallView(arguments[0], + const install_view = new InstallView(0, () => cb_calls.push("show"), () => cb_calls.push("hide")); document.body.append(install_view.main_div); const ets = () => install_view.item_entries; const shw = slice => [cb_calls.slice(slice || 0), install_view.shown]; returnval([container_ids, container_ids.map(cid => install_view[cid])]); - ''', - tab_id) + ''') containers = dict(zip(container_ids, containers_objects)) @@ -60,8 +50,7 @@ def setup_view(driver, execute_in_page): return containers, assert_container_displayed install_ext_data = { - 'content_script': content_script, - 'background_script': background_script, + 'background_script': broker_js, 'extra_html': ExtraHTML('html/install.html', {}), 'navigate_to': 'html/install.html' } @@ -148,7 +137,7 @@ def test_install_normal_usage(driver, execute_in_page, complex_variant): execute_in_page('returnval(install_view.install_but);').click() installed = lambda d: 'ly installed!' in containers['dialog_container'].text - WebDriverWait(driver, 10000).until(installed) + WebDriverWait(driver, 10).until(installed) assert execute_in_page('returnval(shw(2));') == [['show'], True] execute_in_page('returnval(install_view.dialog_ctx.ok_but);').click() @@ -188,10 +177,15 @@ def test_install_normal_usage(driver, execute_in_page, complex_variant): # All items are up to date - verify dialog is instead shown in this case. execute_in_page('install_view.show(...arguments);', 'https://hydril.la/', 'mapping', root_mapping_id) - assert execute_in_page('returnval(shw(6));') == [['show'], True] - assert_container_displayed('dialog_container') + + fetched = lambda d: 'Fetching ' not in containers['dialog_container'].text + WebDriverWait(driver, 10).until(fetched) + assert 'Nothing to do - packages already installed.' \ in containers['dialog_container'].text + assert_container_displayed('dialog_container') + + assert execute_in_page('returnval(shw(6));') == [['show'], True] execute_in_page('returnval(install_view.dialog_ctx.ok_but);').click() assert execute_in_page('returnval(shw(6));') == [['show', 'hide'], False] diff --git a/test/unit/test_repo_query.py b/test/unit/test_repo_query.py new file mode 100644 index 0000000..dd57452 --- /dev/null +++ b/test/unit/test_repo_query.py @@ -0,0 +1,278 @@ +# SPDX-License-Identifier: CC0-1.0 + +""" +Haketilo unit tests - ............. +""" + +# This file is part of Haketilo +# +# Copyright (C) 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the CC0 1.0 Universal License as published by +# the Creative Commons Corporation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# CC0 1.0 Universal License for more details. + +import pytest +import json +from selenium.webdriver.support.ui import WebDriverWait + +from ..extension_crafting import ExtraHTML +from ..script_loader import load_script + +from ..extension_crafting import ExtraHTML +from ..script_loader import load_script +from .utils import * + +repo_urls = [f'https://hydril.la/{s}' for s in ('', '1/', '2/', '3/', '4/')] + +queried_url = 'https://example_a.com/something' + +def setup_view(execute_in_page, repo_urls): + mock_cacher(execute_in_page) + + execute_in_page(load_script('html/repo_query.js')) + execute_in_page( + ''' + const repo_proms = arguments[0].map(url => haketilodb.set_repo(url)); + + const cb_calls = []; + const view = new RepoQueryView(0, + () => cb_calls.push("show"), + () => cb_calls.push("hide")); + document.body.append(view.main_div); + const shw = slice => [cb_calls.slice(slice || 0), view.shown]; + + returnval(Promise.all(repo_proms)); + ''', + repo_urls) + +repo_query_ext_data = { + 'background_script': broker_js, + 'extra_html': ExtraHTML('html/repo_query.html', {}), + 'navigate_to': 'html/repo_query.html' +} + +@pytest.mark.ext_data(repo_query_ext_data) +@pytest.mark.usefixtures('webextension') +def test_repo_query_normal_usage(driver, execute_in_page): + """ + Test of using the repo query view to browse results from repository and to + start installation. + """ + setup_view(execute_in_page, repo_urls) + + assert execute_in_page('returnval(shw());') == [[], False] + + execute_in_page('view.show(arguments[0]);', queried_url) + + assert execute_in_page('returnval(shw());') == [['show'], True] + + def get_repo_entries(driver): + return execute_in_page( + f'returnval((view.repo_entries || []).map({nodes_props_code}));' + ) + + repo_entries = WebDriverWait(driver, 10).until(get_repo_entries) + + assert len(repo_urls) == len(repo_entries) + + for url, entry in reversed(list(zip(repo_urls, repo_entries))): + assert url in entry['main_li'].text + + but_ids = ('show_results_but', 'hide_results_but') + for but_idx in (0, 1, 0): + assert bool(but_idx) == entry['list_container'].is_displayed() + + assert not entry[but_ids[1 - but_idx]].is_displayed() + + entry[but_ids[but_idx]].click() + + def get_mapping_entries(driver): + return execute_in_page( + f'''{{ + const result_entries = (view.repo_entries[0].result_entries || []); + returnval(result_entries.map({nodes_props_code})); + }}''') + + mapping_entries = WebDriverWait(driver, 10).until(get_mapping_entries) + + assert len(mapping_entries) == 3 + + expected_names = ['MAPPING_ABCD', 'MAPPING_ABCD-DEFG-GHIJ', 'MAPPING_A'] + + for name, entry in zip(expected_names, mapping_entries): + assert entry['mapping_name'].text == name + assert entry['mapping_id'].text == f'{name.lower()}-2022.5.11' + + containers = execute_in_page( + '''{ + const reductor = (acc, k) => Object.assign(acc, {[k]: view[k]}); + returnval(container_ids.reduce(reductor, {})); + }''') + + for id, container in containers.items(): + assert (id == 'repos_list_container') == container.is_displayed() + + entry['install_but'].click() + + for id, container in containers.items(): + assert (id == 'install_view_container') == container.is_displayed() + + execute_in_page('returnval(view.install_view.cancel_but);').click() + + for id, container in containers.items(): + assert (id == 'repos_list_container') == container.is_displayed() + + assert execute_in_page('returnval(shw());') == [['show'], True] + execute_in_page('returnval(view.cancel_but);').click() + assert execute_in_page('returnval(shw());') == [['show', 'hide'], False] + +@pytest.mark.ext_data(repo_query_ext_data) +@pytest.mark.usefixtures('webextension') +@pytest.mark.parametrize('message', [ + 'browsing_for', + 'no_repos', + 'failure_to_communicate', + 'HTTP_code', + 'invalid_JSON', + 'newer_API_version', + 'invalid_API_version', + 'querying_repo', + 'no_results' +]) +def test_repo_query_messages(driver, execute_in_page, message): + """ + Test of loading and error messages shown in parts of the repo query view. + """ + def has_msg(message, elem=None): + def has_msg_and_is_visible(dummy_driver): + if elem: + return elem.is_displayed() and message in elem.text + else: + return message in driver.page_source + return has_msg_and_is_visible + + def show_and_wait_for_repo_entry(): + execute_in_page('view.show(arguments[0]);', queried_url) + done = lambda d: execute_in_page('returnval(!!view.repo_entries);') + WebDriverWait(driver, 10).until(done) + execute_in_page( + ''' + if (view.repo_entries.length > 0) + view.repo_entries[0].show_results_but.click(); + ''') + + if message == 'browsing_for': + setup_view(execute_in_page, []) + show_and_wait_for_repo_entry() + + elem = execute_in_page('returnval(view.url_span.parentNode);') + assert has_msg(f'Browsing custom resources for {queried_url}.', elem)(0) + elif message == 'no_repos': + setup_view(execute_in_page, []) + show_and_wait_for_repo_entry() + + elem = execute_in_page('returnval(view.repos_list);') + done = has_msg('You have no repositories configured :(', elem) + WebDriverWait(driver, 10).until(done) + elif message == 'failure_to_communicate': + setup_view(execute_in_page, repo_urls) + execute_in_page( + 'browser.tabs.sendMessage = () => Promise.resolve({error: "sth"});' + ) + show_and_wait_for_repo_entry() + + elem = execute_in_page('returnval(view.repo_entries[0].info_span);') + done = has_msg('Failure to communicate with repository :(', elem) + WebDriverWait(driver, 10).until(done) + elif message == 'HTTP_code': + setup_view(execute_in_page, repo_urls) + execute_in_page( + ''' + const response = {ok: false, status: 405}; + browser.tabs.sendMessage = () => Promise.resolve(response); + ''') + show_and_wait_for_repo_entry() + + elem = execute_in_page('returnval(view.repo_entries[0].info_span);') + done = has_msg('Repository sent HTTP code 405 :(', elem) + WebDriverWait(driver, 10).until(done) + elif message == 'invalid_JSON': + setup_view(execute_in_page, repo_urls) + execute_in_page( + ''' + const response = {ok: true, status: 200, error_json: "sth"}; + browser.tabs.sendMessage = () => Promise.resolve(response); + ''') + show_and_wait_for_repo_entry() + + elem = execute_in_page('returnval(view.repo_entries[0].info_span);') + done = has_msg("Repository's response is not valid JSON :(", elem) + WebDriverWait(driver, 10).until(done) + elif message == 'newer_API_version': + setup_view(execute_in_page, repo_urls) + execute_in_page( + ''' + const response = { + ok: true, + status: 200, + json: {api_schema_version: [1234]} + }; + browser.tabs.sendMessage = () => Promise.resolve(response); + ''') + show_and_wait_for_repo_entry() + + elem = execute_in_page('returnval(view.repo_entries[0].info_span);') + msg = 'Results were served using unsupported Hydrilla API version (1234). You might need to update Haketilo.' + WebDriverWait(driver, 10).until(has_msg(msg, elem)) + elif message == 'invalid_API_version': + setup_view(execute_in_page, repo_urls) + execute_in_page( + ''' + const response = { + ok: true, + status: 200, + json: {api_schema_version: null} + }; + browser.tabs.sendMessage = () => Promise.resolve(response); + ''') + show_and_wait_for_repo_entry() + + elem = execute_in_page('returnval(view.repo_entries[0].info_span);') + msg = 'Results were served using unsupported Hydrilla API version. You might need to update Haketilo.' + WebDriverWait(driver, 10).until(has_msg(msg, elem)) + elif message == 'querying_repo': + setup_view(execute_in_page, repo_urls) + execute_in_page( + 'browser.tabs.sendMessage = () => new Promise(() => {});' + ) + show_and_wait_for_repo_entry() + + elem = execute_in_page('returnval(view.repo_entries[0].info_span);') + assert has_msg('Querying repository...', elem)(0) + elif message == 'no_results': + setup_view(execute_in_page, repo_urls) + execute_in_page( + ''' + const response = { + ok: true, + status: 200, + json: { + api_schema_version: [1], + api_schema_revision: 1, + mappings: [] + } + }; + browser.tabs.sendMessage = () => Promise.resolve(response); + ''') + show_and_wait_for_repo_entry() + + elem = execute_in_page('returnval(view.repo_entries[0].results_list);') + WebDriverWait(driver, 10).until(has_msg('No results :(', elem)) + else: + raise Exception('made a typo in test function params?') diff --git a/test/unit/test_repo_query_cacher.py b/test/unit/test_repo_query_cacher.py index ee9f0fd..b1ce4c8 100644 --- a/test/unit/test_repo_query_cacher.py +++ b/test/unit/test_repo_query_cacher.py @@ -22,7 +22,6 @@ import json from selenium.webdriver.support.ui import WebDriverWait from ..script_loader import load_script -from .utils import * def content_script(): script = load_script('content/repo_query_cacher.js') @@ -39,6 +38,47 @@ def fetch_through_cache(driver, tab_id, url): ''', tab_id, url) +""" +tab_id_responder is meant to be appended to background script of a test +extension. +""" +tab_id_responder = ''' +function tell_tab_id(msg, sender, respond_cb) { + if (msg[0] === "learn_tab_id") + respond_cb(sender.tab.id); +} +browser.runtime.onMessage.addListener(tell_tab_id); +''' + +""" +tab_id_asker is meant to be appended to content script of a test extension. +""" +tab_id_asker = ''' +browser.runtime.sendMessage(["learn_tab_id"]) + .then(tid => window.wrappedJSObject.haketilo_tab = tid); +''' + +def run_content_script_in_new_window(driver, url): + """ + Expect an extension to be loaded which had tab_id_responder and tab_id_asker + appended to its background and content scripts, respectively. + Open the provided url in a new tab, find its tab id and return it, with + current window changed back to the initial one. + """ + initial_handle = driver.current_window_handle + handles = driver.window_handles + driver.execute_script('window.open(arguments[0], "_blank");', url) + WebDriverWait(driver, 10).until(lambda d: d.window_handles is not handles) + new_handle = [h for h in driver.window_handles if h not in handles][0] + + driver.switch_to.window(new_handle) + + get_tab_id = lambda d: d.execute_script('return window.haketilo_tab;') + tab_id = WebDriverWait(driver, 10).until(get_tab_id) + + driver.switch_to.window(initial_handle) + return tab_id + @pytest.mark.ext_data({ 'content_script': content_script, 'background_script': lambda: bypass_js() + ';' + tab_id_responder diff --git a/test/unit/utils.py b/test/unit/utils.py index 90a2ab7..6f0236d 100644 --- a/test/unit/utils.py +++ b/test/unit/utils.py @@ -202,43 +202,49 @@ def are_scripts_allowed(driver, nonce=None): ''', nonce) -""" -tab_id_responder is meant to be appended to background script of a test -extension. -""" -tab_id_responder = ''' -function tell_tab_id(msg, sender, respond_cb) { - if (msg[0] === "learn_tab_id") - respond_cb(sender.tab.id); -} -browser.runtime.onMessage.addListener(tell_tab_id); -''' - -""" -tab_id_asker is meant to be appended to content script of a test extension. -""" -tab_id_asker = ''' -browser.runtime.sendMessage(["learn_tab_id"]) - .then(tid => window.wrappedJSObject.haketilo_tab = tid); -''' - -def run_content_script_in_new_window(driver, url): +def mock_cacher(execute_in_page): """ - Expect an extension to be loaded which had tab_id_responder and tab_id_asker - appended to its background and content scripts, respectively. - Open the provided url in a new tab, find its tab id and return it, with - current window changed back to the initial one. + Some parts of code depend on content/repo_query_cacher.js and + background/CORS_bypass_server.js running in their appropriate contexts. This + function modifies the relevant browser.runtime.sendMessage function to + perform fetch(), bypassing the cacher. """ - initial_handle = driver.current_window_handle - handles = driver.window_handles - driver.execute_script('window.open(arguments[0], "_blank");', url) - WebDriverWait(driver, 10).until(lambda d: d.window_handles is not handles) - new_handle = [h for h in driver.window_handles if h not in handles][0] - - driver.switch_to.window(new_handle) + execute_in_page( + '''{ + const old_sendMessage = browser.tabs.sendMessage, old_fetch = fetch; + async function new_sendMessage(tab_id, msg) { + if (msg[0] !== "repo_query") + return old_sendMessage(tab_id, msg); + + /* Use snapshotted fetch(), allow other test code to override it. */ + const response = await old_fetch(msg[1]); + if (!response) + return {error: "Something happened :o"}; + + const result = {ok: response.ok, status: response.status}; + try { + result.json = await response.json(); + } catch(e) { + result.error_json = "" + e; + } + return result; + } - get_tab_id = lambda d: d.execute_script('return window.haketilo_tab;') - tab_id = WebDriverWait(driver, 10).until(get_tab_id) + browser.tabs.sendMessage = new_sendMessage; + }''') - driver.switch_to.window(initial_handle) - return tab_id +""" +Convenience snippet of code to retrieve a copy of given object with only those +properties present which are DOM nodes. This makes it possible to easily access +DOM nodes stored in a javascript object that also happens to contain some +other properties that make it impossible to return from a Selenium script. +""" +nodes_props_code = '''\ +(obj => { + const result = {}; + for (const [key, value] of Object.entries(obj)) { + if (value instanceof Node) + result[key] = value; + } + return result; +})''' diff --git a/test/world_wide_library.py b/test/world_wide_library.py index 31864fb..b3febd7 100644 --- a/test/world_wide_library.py +++ b/test/world_wide_library.py @@ -107,15 +107,16 @@ def serve_counter(command, get_params, post_params): # Mock a Hydrilla repository. +make_handler = lambda txt: lambda c, g, p: (200, {}, txt) + # Mock files in the repository. sample_contents = [f'Mi povas manĝi vitron, ĝi ne damaĝas min {i}' for i in range(9)] -sample_hashes = [sha256(c.encode()).digest().hex() for c in sample_contents] +sample_hashes = [sha256(c.encode()).digest().hex() for c in sample_contents] -file_url = lambda hashed: f'https://hydril.la/file/sha256-{hashed}' -file_handler = lambda contents: lambda c, g, p: (200, {}, contents) +file_url = lambda hashed: f'https://hydril.la/file/sha256-{hashed}' -sample_files_catalog = dict([(file_url(h), file_handler(c)) +sample_files_catalog = dict([(file_url(h), make_handler(c)) for h, c in zip(sample_hashes, sample_contents)]) # Mock resources and mappings in the repository. @@ -145,6 +146,7 @@ for i in range(10): sample_resources_catalog = {} sample_mappings_catalog = {} +sample_queries = {} for srt in sample_resource_templates: resource = make_sample_resource() @@ -160,8 +162,8 @@ for srt in sample_resource_templates: file_ref = {'file': f'file_{i}', 'sha256': sample_hashes[i]} resource[('source_copyright', 'scripts')[i & 1]].append(file_ref) - # Keeping it simple - just make one corresponding mapping for each resource. - payloads = {'https://example.com/*': {'identifier': resource['identifier']}} + resource_versions = [resource['version'], resource['version'].copy()] + resource_versions[1][-1] += 1 mapping = make_sample_mapping() mapping['api_schema_version'] = [1] @@ -170,20 +172,51 @@ for srt in sample_resource_templates: mapping['long_name'] = mapping['identifier'].upper() mapping['uuid'] = str(uuid4()) mapping['source_copyright'] = resource['source_copyright'] - mapping['payloads'] = payloads - make_handler = lambda txt: lambda c, g, p: (200, {}, txt) + mapping_versions = [mapping['version'], mapping['version'].copy()] + mapping_versions[1][-1] += 1 + + sufs = [srt["id_suffix"], *[l for l in srt["id_suffix"] if l.isalpha()]] + patterns = [f'https://example_{suf}.com/*' for suf in set(sufs)] + payloads = {} + + for pat in patterns: + payloads[pat] = {'identifier': resource['identifier']} + + queryable_url = pat.replace('*', 'something') + if queryable_url not in sample_queries: + sample_queries[queryable_url] = [] + + sample_queries[queryable_url].append({ + 'identifier': mapping['identifier'], + 'long_name': mapping['long_name'], + 'version': mapping_versions[1] + }) - for item, catalog in [ - (resource, sample_resources_catalog), - (mapping, sample_mappings_catalog) + mapping['payloads'] = payloads + + for item, versions, catalog in [ + (resource, resource_versions, sample_resources_catalog), + (mapping, mapping_versions, sample_mappings_catalog) ]: fmt = f'https://hydril.la/{item["type"]}/{item["identifier"]}%s.json' # Make 2 versions of each item so that we can test updates. - for i in range(2): + for ver in versions: + item['version'] = ver for fmt_arg in ('', '/' + item_version_string(item)): catalog[fmt % fmt_arg] = make_handler(json.dumps(item)) - item['version'][-1] += 1 + +def serve_query(command, get_params, post_params): + response = { + 'api_schema_version': [1], + 'api_schema_revision': 1, + 'mappings': sample_queries[get_params['url'][0]] + } + + return (200, {}, json.dumps(response)) + +sample_queries_catalog = dict([(f'https://hydril.la/{suf}query', serve_query) + for suf in ('', '1/', '2/', '3/', '4/')]) catalog = { 'http://gotmyowndoma.in': @@ -233,5 +266,6 @@ catalog = { **sample_files_catalog, **sample_resources_catalog, - **sample_mappings_catalog + **sample_mappings_catalog, + **sample_queries_catalog } -- cgit v1.2.3