add a repo querying HTML interface

6 files changed, 426 insertions, 71 deletions

diff --git a/test/server.py b/test/server.py
index 9cab5cc..0963b5b 100755
--- a/test/server.py
+++ b/test/server.py
@@ -31,6 +31,7 @@ wrapping the classes in proxy_core.py
 from pathlib import Path
 from urllib.parse import parse_qs
 from threading import Thread
+import traceback
 
 from .proxy_core import ProxyRequestHandler, ThreadingHTTPServer
 from .misc_constants import *
@@ -84,8 +85,10 @@ class RequestHijacker(ProxyRequestHandler):
                 status_code, headers = 404, {'Content-Type': 'text/plain'}
                 resp_body = b'Handler for this URL not found.'
 
-        except Exception as e:
-            status_code, headers, resp_body = 500, {'Content-Type': 'text/plain'}, b'Internal Error:\n' + repr(e).encode()
+        except Exception:
+            status_code = 500
+            headers     = {'Content-Type': 'text/plain'}
+            resp_body   = b'Internal Error:\n' + traceback.format_exc().encode()
 
         headers['Content-Length'] = str(len(resp_body))
         self.send_response(status_code)
diff --git a/test/unit/test_install.py b/test/unit/test_install.py
index cb8fe36..303d31a 100644
--- a/test/unit/test_install.py
+++ b/test/unit/test_install.py
@@ -25,31 +25,21 @@ from ..extension_crafting import ExtraHTML
 from ..script_loader import load_script
 from .utils import *
 
-def content_script():
-    script = load_script('content/repo_query_cacher.js')
-    return f'{script}; {tab_id_asker}; start();'
-
-def background_script():
-    script = load_script('background/broadcast_broker.js',
-                         '#IMPORT background/CORS_bypass_server.js')
-    return f'{script}; {tab_id_responder}; start(); CORS_bypass_server.start();'
-
 def setup_view(driver, execute_in_page):
-    tab_id = run_content_script_in_new_window(driver, 'https://gotmyowndoma.in')
+    mock_cacher(execute_in_page)
 
     execute_in_page(load_script('html/install.js'))
     container_ids, containers_objects = execute_in_page(
         '''
         const cb_calls = [];
-        const install_view = new InstallView(arguments[0],
+        const install_view = new InstallView(0,
                                              () => cb_calls.push("show"),
                                              () => cb_calls.push("hide"));
         document.body.append(install_view.main_div);
         const ets = () => install_view.item_entries;
         const shw = slice => [cb_calls.slice(slice || 0), install_view.shown];
         returnval([container_ids, container_ids.map(cid => install_view[cid])]);
-        ''',
-        tab_id)
+        ''')
 
     containers = dict(zip(container_ids, containers_objects))
 
@@ -60,8 +50,7 @@ def setup_view(driver, execute_in_page):
     return containers, assert_container_displayed
 
 install_ext_data = {
-    'content_script': content_script,
-    'background_script': background_script,
+    'background_script': broker_js,
     'extra_html': ExtraHTML('html/install.html', {}),
     'navigate_to': 'html/install.html'
 }
@@ -148,7 +137,7 @@ def test_install_normal_usage(driver, execute_in_page, complex_variant):
 
     execute_in_page('returnval(install_view.install_but);').click()
     installed = lambda d: 'ly installed!' in containers['dialog_container'].text
-    WebDriverWait(driver, 10000).until(installed)
+    WebDriverWait(driver, 10).until(installed)
 
     assert execute_in_page('returnval(shw(2));') == [['show'], True]
     execute_in_page('returnval(install_view.dialog_ctx.ok_but);').click()
@@ -188,10 +177,15 @@ def test_install_normal_usage(driver, execute_in_page, complex_variant):
     # All items are up to date - verify dialog is instead shown in this case.
     execute_in_page('install_view.show(...arguments);',
                     'https://hydril.la/', 'mapping', root_mapping_id)
-    assert execute_in_page('returnval(shw(6));') == [['show'], True]
-    assert_container_displayed('dialog_container')
+
+    fetched = lambda d: 'Fetching ' not in containers['dialog_container'].text
+    WebDriverWait(driver, 10).until(fetched)
+
     assert 'Nothing to do - packages already installed.' \
         in containers['dialog_container'].text
+    assert_container_displayed('dialog_container')
+
+    assert execute_in_page('returnval(shw(6));') == [['show'], True]
     execute_in_page('returnval(install_view.dialog_ctx.ok_but);').click()
     assert execute_in_page('returnval(shw(6));') == [['show', 'hide'], False]
 
diff --git a/test/unit/test_repo_query.py b/test/unit/test_repo_query.py
new file mode 100644
index 0000000..dd57452
--- /dev/null
+++ b/test/unit/test_repo_query.py
@@ -0,0 +1,278 @@
+# SPDX-License-Identifier: CC0-1.0
+
+"""
+Haketilo unit tests - .............
+"""
+
+# This file is part of Haketilo
+#
+# Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the CC0 1.0 Universal License as published by
+# the Creative Commons Corporation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# CC0 1.0 Universal License for more details.
+
+import pytest
+import json
+from selenium.webdriver.support.ui import WebDriverWait
+
+from ..extension_crafting import ExtraHTML
+from ..script_loader import load_script
+
+from ..extension_crafting import ExtraHTML
+from ..script_loader import load_script
+from .utils import *
+
+repo_urls = [f'https://hydril.la/{s}' for s in ('', '1/', '2/', '3/', '4/')]
+
+queried_url = 'https://example_a.com/something'
+
+def setup_view(execute_in_page, repo_urls):
+    mock_cacher(execute_in_page)
+
+    execute_in_page(load_script('html/repo_query.js'))
+    execute_in_page(
+        '''
+        const repo_proms = arguments[0].map(url => haketilodb.set_repo(url));
+
+        const cb_calls = [];
+        const view = new RepoQueryView(0,
+                                       () => cb_calls.push("show"),
+                                       () => cb_calls.push("hide"));
+        document.body.append(view.main_div);
+        const shw = slice => [cb_calls.slice(slice || 0), view.shown];
+
+        returnval(Promise.all(repo_proms));
+        ''',
+        repo_urls)
+
+repo_query_ext_data = {
+    'background_script': broker_js,
+    'extra_html': ExtraHTML('html/repo_query.html', {}),
+    'navigate_to': 'html/repo_query.html'
+}
+
+@pytest.mark.ext_data(repo_query_ext_data)
+@pytest.mark.usefixtures('webextension')
+def test_repo_query_normal_usage(driver, execute_in_page):
+    """
+    Test of using the repo query view to browse results from repository and to
+    start installation.
+    """
+    setup_view(execute_in_page, repo_urls)
+
+    assert execute_in_page('returnval(shw());') == [[], False]
+
+    execute_in_page('view.show(arguments[0]);', queried_url)
+
+    assert execute_in_page('returnval(shw());') == [['show'], True]
+
+    def get_repo_entries(driver):
+        return execute_in_page(
+            f'returnval((view.repo_entries || []).map({nodes_props_code}));'
+        )
+
+    repo_entries = WebDriverWait(driver, 10).until(get_repo_entries)
+
+    assert len(repo_urls) == len(repo_entries)
+
+    for url, entry in reversed(list(zip(repo_urls, repo_entries))):
+        assert url in entry['main_li'].text
+
+    but_ids = ('show_results_but', 'hide_results_but')
+    for but_idx in (0, 1, 0):
+        assert bool(but_idx) == entry['list_container'].is_displayed()
+
+        assert not entry[but_ids[1 - but_idx]].is_displayed()
+
+        entry[but_ids[but_idx]].click()
+
+    def get_mapping_entries(driver):
+        return execute_in_page(
+            f'''{{
+            const result_entries = (view.repo_entries[0].result_entries || []);
+            returnval(result_entries.map({nodes_props_code}));
+            }}''')
+
+    mapping_entries = WebDriverWait(driver, 10).until(get_mapping_entries)
+
+    assert len(mapping_entries) == 3
+
+    expected_names = ['MAPPING_ABCD', 'MAPPING_ABCD-DEFG-GHIJ', 'MAPPING_A']
+
+    for name, entry in zip(expected_names, mapping_entries):
+        assert entry['mapping_name'].text == name
+        assert entry['mapping_id'].text == f'{name.lower()}-2022.5.11'
+
+    containers = execute_in_page(
+        '''{
+        const reductor = (acc, k) => Object.assign(acc, {[k]: view[k]});
+        returnval(container_ids.reduce(reductor, {}));
+        }''')
+
+    for id, container in containers.items():
+        assert (id == 'repos_list_container') == container.is_displayed()
+
+    entry['install_but'].click()
+
+    for id, container in containers.items():
+        assert (id == 'install_view_container') == container.is_displayed()
+
+    execute_in_page('returnval(view.install_view.cancel_but);').click()
+
+    for id, container in containers.items():
+        assert (id == 'repos_list_container') == container.is_displayed()
+
+    assert execute_in_page('returnval(shw());') == [['show'], True]
+    execute_in_page('returnval(view.cancel_but);').click()
+    assert execute_in_page('returnval(shw());') == [['show', 'hide'], False]
+
+@pytest.mark.ext_data(repo_query_ext_data)
+@pytest.mark.usefixtures('webextension')
+@pytest.mark.parametrize('message', [
+    'browsing_for',
+    'no_repos',
+    'failure_to_communicate',
+    'HTTP_code',
+    'invalid_JSON',
+    'newer_API_version',
+    'invalid_API_version',
+    'querying_repo',
+    'no_results'
+])
+def test_repo_query_messages(driver, execute_in_page, message):
+    """
+    Test of loading and error messages shown in parts of the repo query view.
+    """
+    def has_msg(message, elem=None):
+        def has_msg_and_is_visible(dummy_driver):
+            if elem:
+                return elem.is_displayed() and message in elem.text
+            else:
+                return message in driver.page_source
+        return has_msg_and_is_visible
+
+    def show_and_wait_for_repo_entry():
+        execute_in_page('view.show(arguments[0]);', queried_url)
+        done = lambda d: execute_in_page('returnval(!!view.repo_entries);')
+        WebDriverWait(driver, 10).until(done)
+        execute_in_page(
+            '''
+            if (view.repo_entries.length > 0)
+                view.repo_entries[0].show_results_but.click();
+            ''')
+
+    if message == 'browsing_for':
+        setup_view(execute_in_page, [])
+        show_and_wait_for_repo_entry()
+
+        elem = execute_in_page('returnval(view.url_span.parentNode);')
+        assert has_msg(f'Browsing custom resources for {queried_url}.', elem)(0)
+    elif message == 'no_repos':
+        setup_view(execute_in_page, [])
+        show_and_wait_for_repo_entry()
+
+        elem = execute_in_page('returnval(view.repos_list);')
+        done = has_msg('You have no repositories configured :(', elem)
+        WebDriverWait(driver, 10).until(done)
+    elif message == 'failure_to_communicate':
+        setup_view(execute_in_page, repo_urls)
+        execute_in_page(
+            'browser.tabs.sendMessage = () => Promise.resolve({error: "sth"});'
+        )
+        show_and_wait_for_repo_entry()
+
+        elem = execute_in_page('returnval(view.repo_entries[0].info_span);')
+        done = has_msg('Failure to communicate with repository :(', elem)
+        WebDriverWait(driver, 10).until(done)
+    elif message == 'HTTP_code':
+        setup_view(execute_in_page, repo_urls)
+        execute_in_page(
+            '''
+            const response = {ok: false, status: 405};
+            browser.tabs.sendMessage = () => Promise.resolve(response);
+            ''')
+        show_and_wait_for_repo_entry()
+
+        elem = execute_in_page('returnval(view.repo_entries[0].info_span);')
+        done = has_msg('Repository sent HTTP code 405 :(', elem)
+        WebDriverWait(driver, 10).until(done)
+    elif message == 'invalid_JSON':
+        setup_view(execute_in_page, repo_urls)
+        execute_in_page(
+            '''
+            const response = {ok: true, status: 200, error_json: "sth"};
+            browser.tabs.sendMessage = () => Promise.resolve(response);
+            ''')
+        show_and_wait_for_repo_entry()
+
+        elem = execute_in_page('returnval(view.repo_entries[0].info_span);')
+        done = has_msg("Repository's response is not valid JSON :(", elem)
+        WebDriverWait(driver, 10).until(done)
+    elif message == 'newer_API_version':
+        setup_view(execute_in_page, repo_urls)
+        execute_in_page(
+            '''
+            const response = {
+                ok: true,
+                status: 200,
+                json: {api_schema_version: [1234]}
+            };
+            browser.tabs.sendMessage = () => Promise.resolve(response);
+            ''')
+        show_and_wait_for_repo_entry()
+
+        elem = execute_in_page('returnval(view.repo_entries[0].info_span);')
+        msg = 'Results were served using unsupported Hydrilla API version (1234). You might need to update Haketilo.'
+        WebDriverWait(driver, 10).until(has_msg(msg, elem))
+    elif message == 'invalid_API_version':
+        setup_view(execute_in_page, repo_urls)
+        execute_in_page(
+            '''
+            const response = {
+                ok: true,
+                status: 200,
+                json: {api_schema_version: null}
+            };
+            browser.tabs.sendMessage = () => Promise.resolve(response);
+            ''')
+        show_and_wait_for_repo_entry()
+
+        elem = execute_in_page('returnval(view.repo_entries[0].info_span);')
+        msg = 'Results were served using unsupported Hydrilla API version. You might need to update Haketilo.'
+        WebDriverWait(driver, 10).until(has_msg(msg, elem))
+    elif message == 'querying_repo':
+        setup_view(execute_in_page, repo_urls)
+        execute_in_page(
+            'browser.tabs.sendMessage = () => new Promise(() => {});'
+        )
+        show_and_wait_for_repo_entry()
+
+        elem = execute_in_page('returnval(view.repo_entries[0].info_span);')
+        assert has_msg('Querying repository...', elem)(0)
+    elif message == 'no_results':
+        setup_view(execute_in_page, repo_urls)
+        execute_in_page(
+            '''
+            const response = {
+                ok: true,
+                status: 200,
+                json: {
+                    api_schema_version: [1],
+                    api_schema_revision: 1,
+                    mappings: []
+                }
+            };
+            browser.tabs.sendMessage = () => Promise.resolve(response);
+            ''')
+        show_and_wait_for_repo_entry()
+
+        elem = execute_in_page('returnval(view.repo_entries[0].results_list);')
+        WebDriverWait(driver, 10).until(has_msg('No results :(', elem))
+    else:
+        raise Exception('made a typo in test function params?')
diff --git a/test/unit/test_repo_query_cacher.py b/test/unit/test_repo_query_cacher.py
index ee9f0fd..b1ce4c8 100644
--- a/test/unit/test_repo_query_cacher.py
+++ b/test/unit/test_repo_query_cacher.py
@@ -22,7 +22,6 @@ import json
 from selenium.webdriver.support.ui import WebDriverWait
 
 from ..script_loader import load_script
-from .utils import *
 
 def content_script():
     script = load_script('content/repo_query_cacher.js')
@@ -39,6 +38,47 @@ def fetch_through_cache(driver, tab_id, url):
         ''',
         tab_id, url)
 
+"""
+tab_id_responder is meant to be appended to background script of a test
+extension.
+"""
+tab_id_responder = '''
+function tell_tab_id(msg, sender, respond_cb) {
+    if (msg[0] === "learn_tab_id")
+        respond_cb(sender.tab.id);
+}
+browser.runtime.onMessage.addListener(tell_tab_id);
+'''
+
+"""
+tab_id_asker is meant to be appended to content script of a test extension.
+"""
+tab_id_asker = '''
+browser.runtime.sendMessage(["learn_tab_id"])
+    .then(tid => window.wrappedJSObject.haketilo_tab = tid);
+'''
+
+def run_content_script_in_new_window(driver, url):
+    """
+    Expect an extension to be loaded which had tab_id_responder and tab_id_asker
+    appended to its background and content scripts, respectively.
+    Open the provided url in a new tab, find its tab id and return it, with
+    current window changed back to the initial one.
+    """
+    initial_handle = driver.current_window_handle
+    handles = driver.window_handles
+    driver.execute_script('window.open(arguments[0], "_blank");', url)
+    WebDriverWait(driver, 10).until(lambda d: d.window_handles is not handles)
+    new_handle = [h for h in driver.window_handles if h not in handles][0]
+
+    driver.switch_to.window(new_handle)
+
+    get_tab_id = lambda d: d.execute_script('return window.haketilo_tab;')
+    tab_id = WebDriverWait(driver, 10).until(get_tab_id)
+
+    driver.switch_to.window(initial_handle)
+    return tab_id
+
 @pytest.mark.ext_data({
     'content_script': content_script,
     'background_script': lambda: bypass_js() + ';' + tab_id_responder
diff --git a/test/unit/utils.py b/test/unit/utils.py
index 90a2ab7..6f0236d 100644
--- a/test/unit/utils.py
+++ b/test/unit/utils.py
@@ -202,43 +202,49 @@ def are_scripts_allowed(driver, nonce=None):
             ''',
             nonce)
 
-"""
-tab_id_responder is meant to be appended to background script of a test
-extension.
-"""
-tab_id_responder = '''
-function tell_tab_id(msg, sender, respond_cb) {
-    if (msg[0] === "learn_tab_id")
-        respond_cb(sender.tab.id);
-}
-browser.runtime.onMessage.addListener(tell_tab_id);
-'''
-
-"""
-tab_id_asker is meant to be appended to content script of a test extension.
-"""
-tab_id_asker = '''
-browser.runtime.sendMessage(["learn_tab_id"])
-    .then(tid => window.wrappedJSObject.haketilo_tab = tid);
-'''
-
-def run_content_script_in_new_window(driver, url):
+def mock_cacher(execute_in_page):
     """
-    Expect an extension to be loaded which had tab_id_responder and tab_id_asker
-    appended to its background and content scripts, respectively.
-    Open the provided url in a new tab, find its tab id and return it, with
-    current window changed back to the initial one.
+    Some parts of code depend on content/repo_query_cacher.js and
+    background/CORS_bypass_server.js running in their appropriate contexts. This
+    function modifies the relevant browser.runtime.sendMessage function to
+    perform fetch(), bypassing the cacher.
     """
-    initial_handle = driver.current_window_handle
-    handles = driver.window_handles
-    driver.execute_script('window.open(arguments[0], "_blank");', url)
-    WebDriverWait(driver, 10).until(lambda d: d.window_handles is not handles)
-    new_handle = [h for h in driver.window_handles if h not in handles][0]
-
-    driver.switch_to.window(new_handle)
+    execute_in_page(
+        '''{
+        const old_sendMessage = browser.tabs.sendMessage, old_fetch = fetch;
+        async function new_sendMessage(tab_id, msg) {
+            if (msg[0] !== "repo_query")
+                return old_sendMessage(tab_id, msg);
+
+            /* Use snapshotted fetch(), allow other test code to override it. */
+            const response = await old_fetch(msg[1]);
+            if (!response)
+                return {error: "Something happened :o"};
+
+            const result = {ok: response.ok, status: response.status};
+            try {
+                result.json = await response.json();
+            } catch(e) {
+                result.error_json = "" + e;
+            }
+            return result;
+        }
 
-    get_tab_id = lambda d: d.execute_script('return window.haketilo_tab;')
-    tab_id = WebDriverWait(driver, 10).until(get_tab_id)
+        browser.tabs.sendMessage = new_sendMessage;
+        }''')
 
-    driver.switch_to.window(initial_handle)
-    return tab_id
+"""
+Convenience snippet of code to retrieve a copy of given object with only those
+properties present which are DOM nodes. This makes it possible to easily access
+DOM nodes stored in a javascript object that also happens to contain some
+other properties that make it impossible to return from a Selenium script.
+"""
+nodes_props_code = '''\
+(obj => {
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+        if (value instanceof Node)
+            result[key] = value;
+    }
+    return result;
+})'''
diff --git a/test/world_wide_library.py b/test/world_wide_library.py
index 31864fb..b3febd7 100644
--- a/test/world_wide_library.py
+++ b/test/world_wide_library.py
@@ -107,15 +107,16 @@ def serve_counter(command, get_params, post_params):
 
 # Mock a Hydrilla repository.
 
+make_handler = lambda txt: lambda c, g, p: (200, {}, txt)
+
 # Mock files in the repository.
 sample_contents = [f'Mi povas manĝi vitron, ĝi ne damaĝas min {i}'
                    for i in range(9)]
-sample_hashes   = [sha256(c.encode()).digest().hex() for c in sample_contents]
+sample_hashes = [sha256(c.encode()).digest().hex() for c in sample_contents]
 
-file_url     = lambda hashed:   f'https://hydril.la/file/sha256-{hashed}'
-file_handler = lambda contents: lambda c, g, p: (200, {}, contents)
+file_url = lambda hashed: f'https://hydril.la/file/sha256-{hashed}'
 
-sample_files_catalog = dict([(file_url(h), file_handler(c))
+sample_files_catalog = dict([(file_url(h), make_handler(c))
                              for h, c in zip(sample_hashes, sample_contents)])
 
 # Mock resources and mappings in the repository.
@@ -145,6 +146,7 @@ for i in range(10):
 
 sample_resources_catalog = {}
 sample_mappings_catalog = {}
+sample_queries = {}
 
 for srt in sample_resource_templates:
     resource = make_sample_resource()
@@ -160,8 +162,8 @@ for srt in sample_resource_templates:
         file_ref = {'file': f'file_{i}', 'sha256': sample_hashes[i]}
         resource[('source_copyright', 'scripts')[i & 1]].append(file_ref)
 
-    # Keeping it simple - just make one corresponding mapping for each resource.
-    payloads = {'https://example.com/*': {'identifier': resource['identifier']}}
+    resource_versions = [resource['version'], resource['version'].copy()]
+    resource_versions[1][-1] += 1
 
     mapping = make_sample_mapping()
     mapping['api_schema_version']  = [1]
@@ -170,20 +172,51 @@ for srt in sample_resource_templates:
     mapping['long_name']           = mapping['identifier'].upper()
     mapping['uuid']                = str(uuid4())
     mapping['source_copyright']    = resource['source_copyright']
-    mapping['payloads']            = payloads
 
-    make_handler = lambda txt: lambda c, g, p: (200, {}, txt)
+    mapping_versions = [mapping['version'], mapping['version'].copy()]
+    mapping_versions[1][-1] += 1
+
+    sufs = [srt["id_suffix"], *[l for l in srt["id_suffix"] if l.isalpha()]]
+    patterns = [f'https://example_{suf}.com/*' for suf in set(sufs)]
+    payloads = {}
+
+    for pat in patterns:
+        payloads[pat] = {'identifier': resource['identifier']}
+
+        queryable_url = pat.replace('*', 'something')
+        if queryable_url not in sample_queries:
+            sample_queries[queryable_url] = []
+
+        sample_queries[queryable_url].append({
+            'identifier': mapping['identifier'],
+            'long_name':  mapping['long_name'],
+            'version':    mapping_versions[1]
+        })
 
-    for item, catalog in [
-        (resource, sample_resources_catalog),
-        (mapping,  sample_mappings_catalog)
+    mapping['payloads'] = payloads
+
+    for item, versions, catalog in [
+        (resource, resource_versions, sample_resources_catalog),
+        (mapping,  mapping_versions,  sample_mappings_catalog)
     ]:
         fmt = f'https://hydril.la/{item["type"]}/{item["identifier"]}%s.json'
         # Make 2 versions of each item so that we can test updates.
-        for i in range(2):
+        for ver in versions:
+            item['version'] = ver
             for fmt_arg in ('', '/' + item_version_string(item)):
                 catalog[fmt % fmt_arg] = make_handler(json.dumps(item))
-            item['version'][-1] += 1
+
+def serve_query(command, get_params, post_params):
+    response = {
+        'api_schema_version':  [1],
+        'api_schema_revision': 1,
+        'mappings':            sample_queries[get_params['url'][0]]
+    }
+
+    return (200, {}, json.dumps(response))
+
+sample_queries_catalog = dict([(f'https://hydril.la/{suf}query', serve_query)
+                               for suf in ('', '1/', '2/', '3/', '4/')])
 
 catalog = {
     'http://gotmyowndoma.in':
@@ -233,5 +266,6 @@ catalog = {
 
     **sample_files_catalog,
     **sample_resources_catalog,
-    **sample_mappings_catalog
+    **sample_mappings_catalog,
+    **sample_queries_catalog
 }