serialize and deserialize entire Response object when relaying fetch() calls to other contexts using sendMessage

6 files changed, 146 insertions, 141 deletions

diff --git a/test/haketilo_test/unit/test_CORS_bypass_server.py b/test/haketilo_test/unit/test_CORS_bypass_server.py
index 45e4ebb..45f06a9 100644
--- a/test/haketilo_test/unit/test_CORS_bypass_server.py
+++ b/test/haketilo_test/unit/test_CORS_bypass_server.py
@@ -24,29 +24,28 @@ from selenium.webdriver.support.ui import WebDriverWait
 from ..script_loader import load_script
 from ..world_wide_library import some_data
 
-urls = {
-    'resource': 'https://anotherdoma.in/resource/blocked/by/CORS.json',
-    'nonexistent': 'https://nxdoma.in/resource.json',
-    'invalid': 'w3csucks://invalid.url/'
+datas = {
+    'resource':       'https://anotherdoma.in/resource/blocked/by/CORS.json',
+    'nonexistent':    'https://nxdoma.in/resource.json',
+    'invalid':        'w3csucks://invalid.url/',
+    'redirected_ok':  'https://site.with.scripts.block.ed',
+    'redirected_err': 'https://site.with.scripts.block.ed'
 }
 
+for name, url in [*datas.items()]:
+    datas[name] = {'url': url}
+
+datas['redirected_ok']['init']  = {'redirect': 'follow'}
+datas['redirected_err']['init'] = {'redirect': 'error'}
+
 content_script = '''\
-const urls = %s;
-
-function fetch_data(url) {
-    return {
-        url,
-        to_get: ["ok", "status"],
-        to_call: ["text", "json"]
-    };
-}
+const datas = %s;
 
 async function fetch_resources() {
     const results = {};
     const promises = [];
-    for (const [name, url] of Object.entries(urls)) {
-        const sending = browser.runtime.sendMessage(["CORS_bypass",
-                                                     fetch_data(url)]);
+    for (const [name, data] of Object.entries(datas)) {
+        const sending = browser.runtime.sendMessage(["CORS_bypass", data]);
         promises.push(sending.then(response => results[name] = response));
     }
 
@@ -58,7 +57,7 @@ async function fetch_resources() {
 fetch_resources();
 '''
 
-content_script = content_script % json.dumps(urls);
+content_script = content_script % json.dumps(datas);
 
 @pytest.mark.ext_data({
     'content_script': content_script,
@@ -77,33 +76,41 @@ def test_CORS_bypass_server(driver, execute_in_page):
         '''
         const result = {};
         let promises = [];
-        for (const [name, url] of Object.entries(arguments[0])) {
+        for (const [name, data] of Object.entries(arguments[0])) {
             const [ok_cb, err_cb] =
                 ["ok", "err"].map(status => () => result[name] = status);
-            promises.push(fetch(url).then(ok_cb, err_cb));
+            promises.push(fetch(data.url).then(ok_cb, err_cb));
         }
         // Make the promises non-failing.
         promises = promises.map(p => new Promise(cb => p.then(cb, cb)));
         returnval(Promise.all(promises).then(() => result));
         ''',
-        {**urls, 'sameorigin': './nonexistent_resource'})
+        {**datas, 'sameorigin': './nonexistent_resource'})
 
-    assert results == dict([*[(k, 'err') for k in urls.keys()],
+    assert results == dict([*[(k, 'err') for k in datas.keys()],
                             ('sameorigin', 'ok')])
 
     done = lambda d: d.execute_script('return window.haketilo_fetch_results;')
     results = WebDriverWait(driver, 10).until(done)
 
     assert set(results['invalid'].keys()) == {'error'}
+    assert results['invalid']['error']['fileName'].endswith('background.js')
+    assert type(results['invalid']['error']['lineNumber']) is int
+    assert type(results['invalid']['error']['message']) is str
+    assert results['invalid']['error']['name'] == 'TypeError'
 
-    assert set(results['nonexistent'].keys()) == \
-        {'ok', 'status', 'text', 'error_json'}
-    assert results['nonexistent']['ok'] == False
     assert results['nonexistent']['status'] == 404
-    assert results['nonexistent']['text'] == 'Handler for this URL not found.'
+    assert results['nonexistent']['statusText'] == 'Not Found'
+    assert any([name.lower() == 'content-length'
+                for name, value in results['nonexistent']['headers']])
+    assert bytes.fromhex(results['nonexistent']['body']) == \
+        b'Handler for this URL not found.'
 
-    assert set(results['resource'].keys()) == {'ok', 'status', 'text', 'json'}
-    assert results['resource']['ok'] == True
     assert results['resource']['status'] == 200
-    assert results['resource']['text'] == some_data
-    assert results['resource']['json'] == json.loads(some_data)
+    assert results['resource']['statusText'] == 'OK'
+    assert any([name.lower() == 'content-length'
+                for name, value in results['resource']['headers']])
+    assert bytes.fromhex(results['resource']['body']) == b'{"some": "data"}'
+
+    assert results['redirected_ok']['status'] == 200
+    assert results['redirected_err']['error']['name'] == 'TypeError'
diff --git a/test/haketilo_test/unit/test_install.py b/test/haketilo_test/unit/test_install.py
index 1e2063c..29910cf 100644
--- a/test/haketilo_test/unit/test_install.py
+++ b/test/haketilo_test/unit/test_install.py
@@ -26,7 +26,7 @@ from ..script_loader import load_script
 from .utils import *
 
 def setup_view(driver, execute_in_page):
-    mock_cacher(execute_in_page)
+    execute_in_page(mock_cacher_code)
 
     execute_in_page(load_script('html/install.js'))
     container_ids, containers_objects = execute_in_page(
@@ -203,7 +203,6 @@ def test_install_normal_usage(driver, execute_in_page, complex_variant):
     'indexeddb_error_file_uses',
     'failure_to_communicate_fetch',
     'HTTP_code_file',
-    'not_valid_text',
     'sha256_mismatch',
     'indexeddb_error_write'
 ])
@@ -243,7 +242,7 @@ def test_install_dialogs(driver, execute_in_page, message):
     if message == 'fetching_data':
         execute_in_page(
             '''
-            browser.tabs.sendMessage = () => new Promise(cb => {});
+            window.mock_cacher_fetch = () => new Promise(cb => {});
             install_view.show(...arguments);
             ''',
             'https://hydril.la/', 'mapping', 'mapping-a')
@@ -253,7 +252,8 @@ def test_install_dialogs(driver, execute_in_page, message):
     elif message == 'failure_to_communicate_sendmessage':
         execute_in_page(
             '''
-            browser.tabs.sendMessage = () => Promise.resolve({error: "sth"});
+            window.mock_cacher_fetch =
+                () => {throw new Error("Something happened :o")};
             install_view.show(...arguments);
             ''',
             'https://hydril.la/', 'mapping', 'mapping-a')
@@ -262,8 +262,8 @@ def test_install_dialogs(driver, execute_in_page, message):
     elif message == 'HTTP_code_item':
         execute_in_page(
             '''
-            const response = {ok: false, status: 404};
-            browser.tabs.sendMessage = () => Promise.resolve(response);
+            const response = new Response("", {status: 404});
+            window.mock_cacher_fetch = () => Promise.resolve(response);
             install_view.show(...arguments);
             ''',
             'https://hydril.la/', 'mapping', 'mapping-a')
@@ -272,8 +272,8 @@ def test_install_dialogs(driver, execute_in_page, message):
     elif message == 'invalid_JSON':
         execute_in_page(
             '''
-            const response = {ok: true, status: 200, error_json: "sth"};
-            browser.tabs.sendMessage = () => Promise.resolve(response);
+            const response = new Response("sth", {status: 200});
+            window.mock_cacher_fetch = () => Promise.resolve(response);
             install_view.show(...arguments);
             ''',
             'https://hydril.la/', 'mapping', 'mapping-a')
@@ -282,12 +282,11 @@ def test_install_dialogs(driver, execute_in_page, message):
     elif message == 'newer_API_version':
         execute_in_page(
             '''
-            const old_sendMessage = browser.tabs.sendMessage;
-            browser.tabs.sendMessage = async function(...args) {
-                const response = await old_sendMessage(...args);
-                response.json.$schema = "https://hydrilla.koszko.org/schemas/api_mapping_description-255.1.schema.json";
-                return response;
-            }
+            const newer_schema_url =
+                "https://hydrilla.koszko.org/schemas/api_mapping_description-255.1.schema.json";
+            const mocked_json_data = JSON.stringify({$schema: newer_schema_url});
+            const response = new Response(mocked_json_data, {status: 200});
+            window.mock_cacher_fetch = () => Promise.resolve(response);
             install_view.show(...arguments);
             ''',
             'https://hydril.la/', 'mapping', 'mapping-a', [2022, 5, 10])
@@ -297,12 +296,18 @@ def test_install_dialogs(driver, execute_in_page, message):
     elif message == 'invalid_response_format':
         execute_in_page(
             '''
-            const old_sendMessage = browser.tabs.sendMessage;
-            browser.tabs.sendMessage = async function(...args) {
-                const response = await old_sendMessage(...args);
-                /* identifier is not a string as it should be. */
-                response.json.identifier = 1234567;
-                return response;
+            window.mock_cacher_fetch = async function(...args) {
+                const response = await fetch(...args);
+                const json = await response.json();
+
+                /* identifier is no longer a string as it should be. */
+                json.identifier = 1234567;
+
+                return new Response(JSON.stringify(json), {
+                    status:     response.status,
+                    statusText: response.statusText,
+                    headers:    [...response.headers.entries()]
+                });
             }
             install_view.show(...arguments);
             ''',
@@ -352,7 +357,7 @@ def test_install_dialogs(driver, execute_in_page, message):
     elif message == 'failure_to_communicate_fetch':
         execute_in_page(
             '''
-            fetch = () => {throw "some error";};
+            fetch = () => {throw new Error("some error");};
             returnval(install_view.show(...arguments));
             ''',
             'https://hydril.la/', 'mapping', 'mapping-b')
@@ -372,18 +377,6 @@ def test_install_dialogs(driver, execute_in_page, message):
         execute_in_page('returnval(install_view.install_but);').click()
 
         assert_dlg(['conf_buts'], 'Repository sent HTTP code 400 :(')
-    elif message == 'not_valid_text':
-        execute_in_page(
-            '''
-            const err = () => {throw "some error";};
-            fetch = () => Promise.resolve({ok: true, status: 200, text: err});
-            returnval(install_view.show(...arguments));
-            ''',
-            'https://hydril.la/', 'mapping', 'mapping-b')
-
-        execute_in_page('returnval(install_view.install_but);').click()
-
-        assert_dlg(['conf_buts'], "Repository's response is not valid text :(")
     elif message == 'sha256_mismatch':
         execute_in_page(
             '''
diff --git a/test/haketilo_test/unit/test_popup.py b/test/haketilo_test/unit/test_popup.py
index e62feb7..3ef7906 100644
--- a/test/haketilo_test/unit/test_popup.py
+++ b/test/haketilo_test/unit/test_popup.py
@@ -81,28 +81,18 @@ mocked_page_infos = {
 tab_mock_js = '''
 ;
 const mocked_page_info = (%s)[/#mock_page_info-(.*)$/.exec(document.URL)[1]];
+const old_sendMessage = browser.tabs.sendMessage;
 browser.tabs.sendMessage = async function(tab_id, msg) {
     const this_tab_id = (await browser.tabs.getCurrent()).id;
     if (tab_id !== this_tab_id)
         throw `not current tab id (${tab_id} instead of ${this_tab_id})`;
 
-    if (msg[0] === "page_info") {
+    if (msg[0] === "page_info")
         return mocked_page_info;
-    } else if (msg[0] === "repo_query") {
-        const response = await fetch(msg[1]);
-        if (!response)
-            return {error: "Something happened :o"};
-
-        const result = {ok: response.ok, status: response.status};
-        try {
-            result.json = await response.json();
-        } catch(e) {
-            result.error_json = "" + e;
-        }
-        return result;
-    } else {
+    else if (msg[0] === "repo_query")
+        return old_sendMessage(tab_id, msg);
+    else
         throw `bad sendMessage message type: '${msg[0]}'`;
-    }
 }
 
 const old_tabs_query = browser.tabs.query;
@@ -113,6 +103,8 @@ browser.tabs.query = async function(query) {
 }
 ''' % json.dumps(mocked_page_infos)
 
+tab_mock_js = mock_cacher_code + tab_mock_js
+
 popup_ext_data = {
     'background_script': broker_js,
     'extra_html': ExtraHTML(
diff --git a/test/haketilo_test/unit/test_repo_query.py b/test/haketilo_test/unit/test_repo_query.py
index f6cae93..c785406 100644
--- a/test/haketilo_test/unit/test_repo_query.py
+++ b/test/haketilo_test/unit/test_repo_query.py
@@ -29,7 +29,7 @@ repo_urls = [f'https://hydril.la/{s}' for s in ('', '1/', '2/', '3/', '4/')]
 queried_url = 'https://example_a.com/something'
 
 def setup_view(execute_in_page, repo_urls, tab={'id': 0}):
-    mock_cacher(execute_in_page)
+    execute_in_page(mock_cacher_code)
 
     execute_in_page(load_script('html/repo_query.js'))
     execute_in_page(
@@ -185,8 +185,10 @@ def test_repo_query_messages(driver, execute_in_page, message):
     elif message == 'failure_to_communicate':
         setup_view(execute_in_page, repo_urls)
         execute_in_page(
-            'browser.tabs.sendMessage = () => Promise.resolve({error: "sth"});'
-        )
+            '''
+            window.mock_cacher_fetch =
+                () => {throw new Error("Something happened :o")};
+            ''')
         show_and_wait_for_repo_entry()
 
         elem = execute_in_page('returnval(view.repo_entries[0].info_div);')
@@ -196,8 +198,8 @@ def test_repo_query_messages(driver, execute_in_page, message):
         setup_view(execute_in_page, repo_urls)
         execute_in_page(
             '''
-            const response = {ok: false, status: 405};
-            browser.tabs.sendMessage = () => Promise.resolve(response);
+            const response = new Response("", {status: 405});
+            window.mock_cacher_fetch = () => Promise.resolve(response);
             ''')
         show_and_wait_for_repo_entry()
 
@@ -208,8 +210,8 @@ def test_repo_query_messages(driver, execute_in_page, message):
         setup_view(execute_in_page, repo_urls)
         execute_in_page(
             '''
-            const response = {ok: true, status: 200, error_json: "sth"};
-            browser.tabs.sendMessage = () => Promise.resolve(response);
+            const response = new Response("sth", {status: 200});
+            window.mock_cacher_fetch = () => Promise.resolve(response);
             ''')
         show_and_wait_for_repo_entry()
 
@@ -220,12 +222,11 @@ def test_repo_query_messages(driver, execute_in_page, message):
         setup_view(execute_in_page, repo_urls)
         execute_in_page(
             '''
-            const response = {
-                ok: true,
-                status: 200,
-                json: {$schema: "https://hydrilla.koszko.org/schemas/api_query_result-255.2.1.schema.json"}
-            };
-            browser.tabs.sendMessage = () => Promise.resolve(response);
+            const newer_schema_url =
+                "https://hydrilla.koszko.org/schemas/api_query_result-255.2.1.schema.json";
+            const mocked_json_data = JSON.stringify({$schema: newer_schema_url});
+            const response = new Response(mocked_json_data, {status: 200});
+            window.mock_cacher_fetch = () => Promise.resolve(response);
             ''')
         show_and_wait_for_repo_entry()
 
@@ -236,13 +237,19 @@ def test_repo_query_messages(driver, execute_in_page, message):
         setup_view(execute_in_page, repo_urls)
         execute_in_page(
             '''
-            const response = {
-                ok: true,
-                status: 200,
-                /* $schema is not a string as it should be. */
-                json: {$schema: null}
-            };
-            browser.tabs.sendMessage = () => Promise.resolve(response);
+            window.mock_cacher_fetch = async function(...args) {
+                const response = await fetch(...args);
+                const json = await response.json();
+
+                /* $schema is no longer a string as it should be. */
+                json.$schema = null;
+
+                return new Response(JSON.stringify(json), {
+                    status:     response.status,
+                    statusText: response.statusText,
+                    headers:    [...response.headers.entries()]
+                });
+            }
             ''')
         show_and_wait_for_repo_entry()
 
@@ -252,7 +259,7 @@ def test_repo_query_messages(driver, execute_in_page, message):
     elif message == 'querying_repo':
         setup_view(execute_in_page, repo_urls)
         execute_in_page(
-            'browser.tabs.sendMessage = () => new Promise(() => {});'
+            'window.mock_cacher_fetch = () => new Promise(cb => {});'
         )
         show_and_wait_for_repo_entry()
 
@@ -262,15 +269,12 @@ def test_repo_query_messages(driver, execute_in_page, message):
         setup_view(execute_in_page, repo_urls)
         execute_in_page(
             '''
-            const response = {
-                ok: true,
-                status: 200,
-                json: {
-                    $schema: "https://hydrilla.koszko.org/schemas/api_query_result-1.schema.json",
-                    mappings: []
-                }
-            };
-            browser.tabs.sendMessage = () => Promise.resolve(response);
+            const schema_url =
+                "https://hydrilla.koszko.org/schemas/api_query_result-1.schema.json";
+            const mocked_json_data =
+                JSON.stringify({$schema: schema_url, mappings: []});
+            const response = new Response(mocked_json_data, {status: 200});
+            window.mock_cacher_fetch = () => Promise.resolve(response);
             ''')
         show_and_wait_for_repo_entry()
 
diff --git a/test/haketilo_test/unit/test_repo_query_cacher.py b/test/haketilo_test/unit/test_repo_query_cacher.py
index 5fbc5cd..3f0a00d 100644
--- a/test/haketilo_test/unit/test_repo_query_cacher.py
+++ b/test/haketilo_test/unit/test_repo_query_cacher.py
@@ -85,34 +85,35 @@ def run_content_script_in_new_window(driver, url):
     'background_script': lambda: bypass_js() + ';' + tab_id_responder
 })
 @pytest.mark.usefixtures('webextension')
-def test_repo_query_cacher_normal_use(driver, execute_in_page):
+def test_repo_query_cacher_normal_use(driver):
     """
     Test if HTTP requests made through our cacher return correct results.
     """
     tab_id = run_content_script_in_new_window(driver, 'https://gotmyowndoma.in')
 
     result = fetch_through_cache(driver, tab_id, 'https://counterdoma.in/')
-    assert set(result.keys()) == {'ok', 'status', 'json'}
-    counter_initial = result['json']['counter']
+    assert set(result.keys()) == {'status', 'statusText', 'headers', 'body'}
+    counter_initial = json.loads(bytes.fromhex(result['body']))['counter']
     assert type(counter_initial) is int
 
     for i in range(2):
         result = fetch_through_cache(driver, tab_id, 'https://counterdoma.in/')
-        assert result['json']['counter'] == counter_initial
+        assert json.loads(bytes.fromhex(result['body'])) \
+            == {'counter': counter_initial}
 
     tab_id = run_content_script_in_new_window(driver, 'https://gotmyowndoma.in')
     result = fetch_through_cache(driver, tab_id, 'https://counterdoma.in/')
-    assert result['json']['counter'] == counter_initial + 1
+    assert json.loads(bytes.fromhex(result['body'])) \
+        == {'counter': counter_initial + 1}
 
     for i in range(2):
         result = fetch_through_cache(driver, tab_id, 'https://nxdoma.in/')
-        assert set(result.keys()) == {'ok', 'status', 'error_json'}
-        assert result['ok'] == False
         assert result['status'] == 404
 
     for i in range(2):
         result = fetch_through_cache(driver, tab_id, 'bad://url')
         assert set(result.keys()) == {'error'}
+        assert result['error']['name'] == 'TypeError'
 
 @pytest.mark.ext_data({
     'content_script': content_script,
@@ -128,3 +129,7 @@ def test_repo_query_cacher_bgscript_error(driver):
 
     result = fetch_through_cache(driver, tab_id, 'https://counterdoma.in/')
     assert set(result.keys()) == {'error'}
+    assert set(result['error'].keys()) == \
+        {'name', 'message', 'fileName', 'lineNumber'}
+    assert result['error']['message'] == \
+        "Couldn't communicate with background script."
diff --git a/test/haketilo_test/unit/utils.py b/test/haketilo_test/unit/utils.py
index 7ddf92a..a49ce8c 100644
--- a/test/haketilo_test/unit/utils.py
+++ b/test/haketilo_test/unit/utils.py
@@ -246,36 +246,40 @@ def mock_broadcast(execute_in_page):
         'Object.keys(broadcast).forEach(k => broadcast[k] = () => {});'
     )
 
-def mock_cacher(execute_in_page):
-    """
-    Some parts of code depend on content/repo_query_cacher.js and
-    background/CORS_bypass_server.js running in their appropriate contexts. This
-    function modifies the relevant browser.runtime.sendMessage function to
-    perform fetch(), bypassing the cacher.
-    """
-    execute_in_page(
-        '''{
-        const old_sendMessage = browser.tabs.sendMessage, old_fetch = fetch;
-        async function new_sendMessage(tab_id, msg) {
-            if (msg[0] !== "repo_query")
-                return old_sendMessage(tab_id, msg);
+"""
+Some parts of code depend on content/repo_query_cacher.js and
+background/CORS_bypass_server.js running in their appropriate contexts. This
+snippet modifies the relevant browser.runtime.sendMessage function to perform
+fetch(), bypassing the cacher.
+"""
+mock_cacher_code = '''{
+const uint8_to_hex =
+    array => [...array].map(b => ("0" + b.toString(16)).slice(-2)).join("");
 
-            /* Use snapshotted fetch(), allow other test code to override it. */
-            const response = await old_fetch(msg[1]);
-            if (!response)
-                return {error: "Something happened :o"};
+const old_sendMessage = browser.tabs.sendMessage;
+window.mock_cacher_fetch = fetch;
+browser.tabs.sendMessage = async function(tab_id, msg) {
+    if (msg[0] !== "repo_query")
+        return old_sendMessage(tab_id, msg);
 
-            const result = {ok: response.ok, status: response.status};
-            try {
-                result.json = await response.json();
-            } catch(e) {
-                result.error_json = "" + e;
-            }
-            return result;
+    /*
+     * Use snapshotted fetch() under the name window.mock_cacher_fetch,
+     * allow other test code to override it.
+     */
+    try {
+        const response = await window.mock_cacher_fetch(msg[1]);
+        const buf = await response.arrayBuffer();
+        return {
+            status:     response.status,
+            statusText: response.statusText,
+            headers:    [...response.headers.entries()],
+            body:       uint8_to_hex(new Uint8Array(buf))
         }
-
-        browser.tabs.sendMessage = new_sendMessage;
-        }''')
+    } catch(e) {
+        return {error: {name: e.name, message: e.message}};
+    }
+}
+}'''
 
 """
 Convenience snippet of code to retrieve a copy of given object with only those