allow eval() in injected scripts

9 files changed, 124 insertions, 56 deletions

diff --git a/common/entities.js b/common/entities.js
index 74cad20..41d6e3b 100644
--- a/common/entities.js
+++ b/common/entities.js
@@ -116,6 +116,28 @@ function* get_used_files(item)
 }
 #EXPORT get_used_files AS get_files
 
+/*
+ * Regex to parse URIs like:
+ *     https://hydrilla.koszko.org/schemas/api_mapping_description-2.schema.json
+ */
+const name_base_re    = "(?<name_base>[^/]*)";
+const major_number_re = "(?<major>[1-9][0-9]*)";
+const minor_number_re = "(?:[1-9][0-9]*|0)";
+const numbers_rest_re = `(?:\\.${minor_number_re})*`;
+const version_re      = `(?<ver>${major_number_re}${numbers_rest_re})`;
+const schema_name_re  = `${name_base_re}-${version_re}\\.schema\\.json`;
+
+const haketilo_schema_name_regex = new RegExp(schema_name_re);
+#EXPORT haketilo_schema_name_regex
+
+/* Extract the number that indicates entity's compatibility mode. */
+function get_schema_major_version(instance) {
+    const match = haketilo_schema_name_regex.exec(instance.$schema);
+
+    return parseInt(match.groups.major);
+}
+#EXPORT get_schema_major_version
+
 #IF NEVER
 
 /*
diff --git a/common/jsonschema.js b/common/jsonschema.js
index 3e99cd6..9c4a70c 100644
--- a/common/jsonschema.js
+++ b/common/jsonschema.js
@@ -57,6 +57,8 @@
 
 #FROM common/jsonschema/scan.js IMPORT SchemaScanResult, scan
 
+#FROM common/entities.js IMPORT haketilo_schema_name_regex
+
 #EXPORT scan
 #EXPORT SchemaScanResult
 
@@ -86,15 +88,6 @@ const haketilo_schemas = [
 #INCLUDE schemas/2.x/common_definitions-2.schema.json
 ].reduce((ac, s) => Object.assign(ac, {[s.$id]: s}), {});
 
-const name_base_re    = "(?<name_base>[^/]*)";
-const major_number_re = "(?<major>[1-9][0-9]*)";
-const minor_number_re = "(?:[1-9][0-9]*|0)";
-const numbers_rest_re = `(?:\\.${minor_number_re})*`;
-const version_re      = `(?<ver>${major_number_re}${numbers_rest_re})`;
-const schema_name_re  = `${name_base_re}-${version_re}\\.schema\\.json`;
-
-const haketilo_schema_name_regex = new RegExp(schema_name_re);
-
 for (const [$id, schema] of [...Object.entries(haketilo_schemas)]) {
     const match = haketilo_schema_name_regex.exec($id);
     const schema_name =
@@ -103,7 +96,6 @@ for (const [$id, schema] of [...Object.entries(haketilo_schemas)]) {
 }
 
 #EXPORT haketilo_schemas
-#EXPORT haketilo_schema_name_regex
 
 const haketilo_validator = new Validator();
 Object.values(haketilo_schemas)
diff --git a/common/policy.js b/common/policy.js
index e14d8cd..6bcb54b 100644
--- a/common/policy.js
+++ b/common/policy.js
@@ -49,16 +49,15 @@
  * CSP rule that either blocks all scripts or only allows scripts with specified
  * nonce attached.
  */
-function make_csp(nonce)
-{
-    const rule = nonce ? `nonce-${nonce}` : "none";
+function make_csp(nonce) {
+    const rule = nonce ? `'nonce-${nonce}'` : "'none'";
     const csp_list = [
-	["prefetch-src",    "none"],
-	["script-src-attr", "none"],
-	["script-src",      rule],
+	["prefetch-src",    "'none'"],
+	["script-src-attr", "'none'"],
+	["script-src",      rule, "'unsafe-eval'"],
 	["script-src-elem", rule]
     ];
-    return csp_list.map(([a, b]) => `${a} '${b}';`).join(" ");
+    return csp_list.map(words => `${words.join(" ")};`).join(" ");
 }
 
 function decide_policy(patterns_tree, url, default_allow, secret)
@@ -113,3 +112,33 @@ function decide_policy(patterns_tree, url, default_allow, secret)
 #EXPORT decide_policy
 
 #EXPORT  () => ({allow: false, csp: make_csp()})  AS fallback_policy
+
+#IF NEVER
+
+/*
+ * Note: the functions below were overeagerly written and are not used now but
+ * might prove useful to once we add more functionalities and are hence kept...
+ */
+
+function relaxed_csp_eval(csp) {
+    const new_csp_list = [];
+
+    for (const directive of csp.split(";")) {
+	const directive_words = directive.trim().split(" ");
+	if (directive_words[0] === "script-src")
+	    directive_words.push("'unsafe-eval'");
+
+	new_csp_list.push(directive_words);
+    }
+
+    new_policy.csp = new_csp_list.map(d => `${d.join(" ")}';`).join(" ");
+}
+
+function relax_policy_eval(policy) {
+    const new_policy = Object.assign({}, policy);
+
+    return Object.assign(new_policy, {csp: relaxed_csp_eval(policy.csp)});
+}
+#EXPORT relax_policy_eval
+
+#ENDIF
diff --git a/html/install.js b/html/install.js
index 5fcf879..82df661 100644
--- a/html/install.js
+++ b/html/install.js
@@ -49,8 +49,8 @@
 #FROM html/DOM_helpers.js  IMPORT clone_template, Showable
 #FROM common/entities.js   IMPORT item_id_string, version_string, get_files
 #FROM common/misc.js       IMPORT sha256_async AS compute_sha256
-#FROM common/jsonschema.js IMPORT haketilo_validator, haketilo_schemas, \
-                                  haketilo_schema_name_regex
+#FROM common/jsonschema.js IMPORT haketilo_validator, haketilo_schemas
+#FROM common/entities.js   IMPORT haketilo_schema_name_regex
 
 #FROM html/repo_query_cacher_client.js IMPORT indirect_fetch
 
diff --git a/test/haketilo_test/unit/test_policy_deciding.py b/test/haketilo_test/unit/test_policy_deciding.py
index 75b35ac..1be488f 100644
--- a/test/haketilo_test/unit/test_policy_deciding.py
+++ b/test/haketilo_test/unit/test_policy_deciding.py
@@ -23,19 +23,36 @@ import pytest
 
 from ..script_loader import load_script
 
-csp_re = re.compile(r'^\S+\s+\S+;(?:\s+\S+\s+\S+;)*$')
-rule_re = re.compile(r'^\s*(?P<src_kind>\S+)\s+(?P<allowed_origins>\S+)$')
+csp_re = re.compile(r'''
+^
+\S+(?:\s+\S+)+;      # first directive
+(?:
+  \s+\S+(?:\s+\S+)+; # subsequent directive
+)*
+$
+''',
+re.VERBOSE)
+
+rule_re = re.compile(r'''
+^
+\s*
+(?P<src_kind>\S+)
+\s+
+(?P<allowed_origins>
+  \S+(?:\s+\S+)*
+)
+$
+''', re.VERBOSE)
+
 def parse_csp(csp):
-    '''
-    Parsing of CSP string into a dict. A simplified format of CSP is assumed.
-    '''
+    '''Parsing of CSP string into a dict.'''
     assert csp_re.match(csp)
 
     result = {}
 
     for rule in csp.split(';')[:-1]:
         match = rule_re.match(rule)
-        result[match.group('src_kind')] = match.group('allowed_origins')
+        result[match.group('src_kind')] = match.group('allowed_origins').split()
 
     return result
 
@@ -78,10 +95,10 @@ def test_decide_policy(execute_in_page):
     for prop in ('mapping', 'payload', 'nonce', 'error'):
         assert prop not in policy
     assert parse_csp(policy['csp']) == {
-        'prefetch-src':    "'none'",
-        'script-src-attr': "'none'",
-        'script-src':      "'none'",
-        'script-src-elem': "'none'"
+        'prefetch-src':    ["'none'"],
+        'script-src-attr': ["'none'"],
+        'script-src':      ["'none'", "'unsafe-eval'"],
+        'script-src-elem': ["'none'"]
     }
 
     policy = execute_in_page(
@@ -95,10 +112,10 @@ def test_decide_policy(execute_in_page):
     for prop in ('payload', 'nonce', 'error'):
         assert prop not in policy
     assert parse_csp(policy['csp']) == {
-        'prefetch-src':    "'none'",
-        'script-src-attr': "'none'",
-        'script-src':      "'none'",
-        'script-src-elem': "'none'"
+        'prefetch-src':    ["'none'"],
+        'script-src-attr': ["'none'"],
+        'script-src':      ["'none'", "'unsafe-eval'"],
+        'script-src-elem': ["'none'"]
     }
 
     policy = execute_in_page(
@@ -114,10 +131,10 @@ def test_decide_policy(execute_in_page):
     assert policy['nonce'] == \
         sha256('m1:res1:http://kno.wn/:abcd'.encode()).digest().hex()
     assert parse_csp(policy['csp']) == {
-        'prefetch-src':    f"'none'",
-        'script-src-attr': f"'none'",
-        'script-src':      f"'nonce-{policy['nonce']}'",
-        'script-src-elem': f"'nonce-{policy['nonce']}'"
+        'prefetch-src':    ["'none'"],
+        'script-src-attr': ["'none'"],
+        'script-src':      [f"'nonce-{policy['nonce']}'", "'unsafe-eval'"],
+        'script-src-elem': [f"'nonce-{policy['nonce']}'"]
     }
 
     policy = execute_in_page(
@@ -128,8 +145,8 @@ def test_decide_policy(execute_in_page):
     for prop in ('mapping', 'payload', 'nonce'):
         assert prop not in policy
     assert parse_csp(policy['csp']) == {
-        'prefetch-src':    "'none'",
-        'script-src-attr': "'none'",
-        'script-src':      "'none'",
-        'script-src-elem': "'none'"
+        'prefetch-src':    ["'none'"],
+        'script-src-attr': ["'none'"],
+        'script-src':      ["'none'", "'unsafe-eval'"],
+        'script-src-elem': ["'none'"]
     }
diff --git a/test/haketilo_test/unit/test_policy_enforcing.py b/test/haketilo_test/unit/test_policy_enforcing.py
index 90a6ed9..4bc6470 100644
--- a/test/haketilo_test/unit/test_policy_enforcing.py
+++ b/test/haketilo_test/unit/test_policy_enforcing.py
@@ -31,12 +31,12 @@ nonce = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
 allow_policy = {'allow': True}
 block_policy = {
     'allow': False,
-    'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'none'; script-src-elem 'none'; frame-src http://* https://*;"
+    'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'none' 'unsafe-eval'; script-src-elem 'none'; frame-src http://* https://*;"
 }
 payload_policy = {
     'mapping': 'somemapping',
     'payload': {'identifier': 'someresource'},
-    'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'nonce-{nonce}'; script-src-elem 'nonce-{nonce}';"
+    'csp': f"prefetch-src 'none'; script-src-attr 'none'; script-src 'nonce-{nonce}' 'unsafe-eval'; script-src-elem 'nonce-{nonce}';"
 }
 
 def content_script():
diff --git a/test/haketilo_test/unit/test_webrequest.py b/test/haketilo_test/unit/test_webrequest.py
index 1244117..dc329b8 100644
--- a/test/haketilo_test/unit/test_webrequest.py
+++ b/test/haketilo_test/unit/test_webrequest.py
@@ -85,7 +85,7 @@ nonce = f'nonce-{sha256(nonce_source).digest().hex()}'
 payload_csp_header = {
     'name': f'Content-Security-Policy',
     'value': ("prefetch-src 'none'; script-src-attr 'none'; "
-              f"script-src '{nonce}'; script-src-elem '{nonce}';")
+              f"script-src '{nonce}' 'unsafe-eval'; script-src-elem '{nonce}';")
 }
 
 sample_payload_headers = [
@@ -107,7 +107,7 @@ sample_blocked_headers.append(sample_csp_header)
 sample_blocked_headers.append({
     'name': f'Content-Security-Policy',
     'value': ("prefetch-src 'none'; script-src-attr 'none'; "
-              f"script-src 'none'; script-src-elem 'none';")
+              "script-src 'none' 'unsafe-eval'; script-src-elem 'none';")
 })
 
 @pytest.mark.get_page('https://gotmyowndoma.in')
diff --git a/test/haketilo_test/unit/utils.py b/test/haketilo_test/unit/utils.py
index a49ce8c..9b3e4a0 100644
--- a/test/haketilo_test/unit/utils.py
+++ b/test/haketilo_test/unit/utils.py
@@ -228,12 +228,21 @@ def are_scripts_allowed(driver, nonce=None):
         return driver.execute_script(
             '''
             document.haketilo_scripts_allowed = false;
+            document.haketilo_eval_allowed = false;
             const html_ns = "http://www.w3.org/1999/xhtml";
             const script = document.createElementNS(html_ns, "script");
-            script.innerHTML = "document.haketilo_scripts_allowed = true;";
+            script.innerHTML = `
+                document.haketilo_scripts_allowed = true;
+                eval('document.haketilo_eval_allowed = true;');
+            `;
             if (arguments[0])
                 script.setAttribute("nonce", arguments[0]);
             (document.head || document.documentElement).append(script);
+
+            if (document.haketilo_scripts_allowed !=
+                document.haketilo_eval_allowed)
+                throw "scripts allowed but eval blocked";
+
             return document.haketilo_scripts_allowed;
             ''',
             nonce)
diff --git a/test/haketilo_test/world_wide_library.py b/test/haketilo_test/world_wide_library.py
index 2d227dd..92ce97e 100644
--- a/test/haketilo_test/world_wide_library.py
+++ b/test/haketilo_test/world_wide_library.py
@@ -152,7 +152,8 @@ sample_resource_templates.append({
     'id_suffix':         'a-w-required-mapping-v1',
     'files_count':       1,
     'dependencies':      [],
-    'required_mappings': [{'identifier': 'mapping-a'}]
+    'required_mappings': [{'identifier': 'mapping-a'}],
+    'include_in_query':  False
 })
 
 sample_resource_templates.append({
@@ -160,7 +161,8 @@ sample_resource_templates.append({
     'files_count':       1,
     'dependencies':      [],
     'required_mappings': [{'identifier': 'mapping-a'}],
-    'schema_ver':        '2'
+    'schema_ver':        '2',
+    'include_in_query':  False
 })
 
 sample_resources_catalog = {}
@@ -193,23 +195,20 @@ for srt in sample_resource_templates:
 
     sufs = [srt["id_suffix"], *[l for l in srt["id_suffix"] if l.isalpha()]]
     patterns = [f'https://example_{suf}.com/*' for suf in set(sufs)]
-    payloads = {}
+    mapping['payloads'] = {}
 
     for pat in patterns:
-        payloads[pat] = {'identifier': resource['identifier']}
+        mapping['payloads'][pat] = {'identifier': resource['identifier']}
 
-        queryable_url = pat.replace('*', 'something')
-        if queryable_url not in sample_queries:
-            sample_queries[queryable_url] = []
+        if not srt.get('include_in_query', True):
+            continue
 
-        sample_queries[queryable_url].append({
+        sample_queries.setdefault(pat.replace('*', 'something'), []).append({
             'identifier': mapping['identifier'],
             'long_name':  mapping['long_name'],
             'version':    mapping_versions[1]
         })
 
-    mapping['payloads'] = payloads
-
     for item in resource, mapping:
         if 'required_mappings' in srt:
             item['required_mappings'] = srt['required_mappings']