[proxy] add documentation page describing URL patterns in Haketilo

1 files changed, 312 insertions, 23 deletions

diff --git a/src/hydrilla/locales/en_US/LC_MESSAGES/messages.po b/src/hydrilla/locales/en_US/LC_MESSAGES/messages.po
index 43e6d18..35e4646 100644
--- a/src/hydrilla/locales/en_US/LC_MESSAGES/messages.po
+++ b/src/hydrilla/locales/en_US/LC_MESSAGES/messages.po
@@ -1,13 +1,20 @@
-# SPDX-License-Identifier: CC0-1.0
+# SPDX-License-Identifier: GPL-3.0-or-later OR CC-BY-SA-4.0
+# English translations for Hydrilla&Haketilo.
 #
-# English (United States) translations for hydrilla.
-# Copyright (C) 2021, 2022 Wojtek Kosior <koszko@koszko.org>
-# Available under the terms of Creative Commons Zero v1.0 Universal.
+# This file is part of Hydrilla&Haketilo.
+# Copyright (C) 2021-2022 Wojtek Kosior <koszko@koszko.org>
+# Dual licensed under
+# * GNU General Public License v3.0 or later and
+# * Creative Commons Attribution Share Alike 4.0 International.
+# You can choose to use either of these licenses or both.
+# I, Wojtek Kosior, thereby promise not to sue for violation of this
+# file's licenses. Although I request that you do not make use of this
+# code in a proprietary work, I am not going to enforce this in court.
 msgid ""
 msgstr ""
 "Project-Id-Version: hydrilla 2.0\n"
 "Report-Msgid-Bugs-To: koszko@koszko.org\n"
-"POT-Creation-Date: 2022-10-25 10:28+0200\n"
+"POT-Creation-Date: 2022-10-26 14:26+0200\n"
 "PO-Revision-Date: 2022-02-12 00:00+0000\n"
 "Last-Translator: Wojtek Kosior <koszko@koszko.org>\n"
 "Language: en_US\n"
@@ -213,39 +220,39 @@ msgstr ""
 "\n"
 "{}"
 
-#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:25
+#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:34
 msgid "info.base.title"
 msgstr "Page info"
 
-#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:30
+#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:39
 msgid "info.base.heading.page_info"
 msgstr "Haketilo page handling details"
 
-#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:33
+#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:42
 msgid "info.base.page_url_label"
 msgstr "Page URL"
 
-#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:41
+#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:50
 msgid "info.base.page_policy_label"
 msgstr "Active policy"
 
-#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:53
+#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:62
 msgid "info.base.more_config_options_label"
 msgstr "Configure"
 
-#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:61
+#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:70
 msgid "info.base.this_site_script_blocking_button"
 msgstr "JS blocking on this site"
 
-#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:64
+#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:73
 msgid "info.base.this_site_payload_button"
 msgstr "Payload for this site"
 
-#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:67
+#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:76
 msgid "info.base.this_page_script_blocking_button"
 msgstr "JS blocking on this page"
 
-#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:70
+#: src/hydrilla/proxy/policies/info_pages_templates/info_base.html.jinja:79
 msgid "info.base.this_page_payload_button"
 msgstr "Payload for this page"
 
@@ -312,6 +319,288 @@ msgstr "Requested file could not be found."
 msgid "api.resource_not_enabled_for_access"
 msgstr "Requested resource is not enabled for access."
 
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:30
+msgid "doc.url_patterns.html.intro"
+msgstr ""
+"We want to be able to apply different rules and custom scripts for "
+"different websites. However, merely specifying \"do this for all "
+"documents under <code>https://example.com</code>\" is not enough. Single "
+"site's pages might differ strongly and require different custom scripts "
+"to be loaded. Always matching against a full URL like "
+"<code>https://example.com/something/somethingelse</code> is also not a "
+"good option. It doesn't allow us to properly handle a site that serves "
+"similar pages for multiple values substituted for "
+"<code>somethingelse</code>."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:35
+msgid "doc.url_patterns.heading.employed_solution"
+msgstr "Employed solution"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:38
+msgid "doc.url_patterns.html.wildcards_intro"
+msgstr ""
+"Wildcards are being used to address the problem. Each payload and rule in"
+" Haketilo has a URL pattern that specifies to which internet pages it "
+"applies. A URL pattern can be as as simple as literal URL in which case "
+"it only matches itself. It can also contain wildcards in the form of one "
+"or more asterisks (<code>*</code>) that correspond to multiple possible "
+"strings occurring in that place."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:42
+msgid "doc.url_patterns.html.wildcards_types_introduced"
+msgstr ""
+"Wildcards can appear in URL's domain and path that follows it. These 2 "
+"types of wildcards are handled separately."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:47
+msgid "doc.url_patterns.label.domain_wildcards"
+msgstr "Domain wildcards"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:50
+msgid "doc.url_patterns.html.domain_wildcards_intro"
+msgstr ""
+"A domain wildcard takes the form of one, two or three asterisks occurring"
+" in place of a single domain name segment at the beginning (left). "
+"Depending on the number of asterisks, the meaning is as follows"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:55
+msgid "doc.url_patterns.html.domain_no_asterisks_example"
+msgstr ""
+"no asterisks (e.g. <code>example.com</code>) - match domain name exactly "
+"(e.g. <code>example.com</code>)"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:58
+msgid "doc.url_patterns.html.domain_one_asterisk_example"
+msgstr ""
+"one asterisk (e.g. <code>*.example.com</code>) - match all domains "
+"resulting from substituting <code>*</code> with a <span "
+"class=\"bold\">single</span> segment (e.g. "
+"<code>banana.example.com</code> or <code>pineapple.example.com</code> but"
+" <span class=\"bold\">not</span> <code>pineapple.pen.example.com</code> "
+"nor <code>example.com</code>)"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:61
+msgid "doc.url_patterns.html.domain_two_asterisks_example"
+msgstr ""
+"two asterisks (e.g. <code>**.example.com</code>) - match all domains "
+"resulting from substituting <code>**</code> with <span class=\"bold\">two"
+" or more</span> segments (e.g. <code>monad.breakfast.example.com</code> "
+"or <code>pure.monad.breakfast.example.com</code> but <span "
+"class=\"bold\">not</span> <code>cabalhell.example.com</code> nor "
+"<code>example.com</code>)"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:64
+msgid "doc.url_patterns.html.domain_three_asterisks_example"
+msgstr ""
+"three asterisks (e.g. <code>***.example.com</code>) - match all domains "
+"resulting from substituting <code>***</code> with <span "
+"class=\"bold\">zero or more</span> segments (e.g. "
+"<code>hello.parkmeter.example.com</code> or "
+"<code>iliketrains.example.com</code> or <code>example.com</code>)"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:70
+msgid "doc.url_patterns.label.path_wildcards"
+msgstr "Path wildcards"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:73
+msgid "doc.url_patterns.html.path_wildcards_intro"
+msgstr ""
+"A path wildcard takes the form of one, two or three asterisks occurring "
+"in place of a single path segment at the end of path (right). Depending "
+"on the number of asterisks, the meaning is as follows"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:78
+msgid "doc.url_patterns.html.path_no_asterisks_example"
+msgstr ""
+"no asterisks (e.g. <code>/joke/clowns</code>) - match path exactly (e.g. "
+"<code>/joke/clowns</code>)"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:81
+msgid "doc.url_patterns.html.path_one_asterisk_example"
+msgstr ""
+"one asterisk (e.g. <code>/itscalled/*</code>) - match all paths resulting"
+" from substituting <code>*</code> with a <span "
+"class=\"bold\">single</span> segment (e.g. "
+"<code>/itscalled/gnulinux</code> or <code>/itscalled/glamp</code> but "
+"<span class=\"bold\">not</span> <code>/itscalled/</code> nor "
+"<code>/itscalled/gnu/linux</code>)"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:84
+msgid "doc.url_patterns.html.path_two_asterisks_example"
+msgstr ""
+"two asterisks (e.g. <code>/another/**</code>) - match all paths resulting"
+" from substituting <code>**</code> with <span class=\"bold\">two or "
+"more</span> segments (e.g. <code>/another/nsa/backdoor</code> or "
+"<code>/another/best/programming/language</code> but <span "
+"class=\"bold\">not</span> <code>/another/apibreak</code> nor "
+"<code>/another</code>)"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:87
+msgid "doc.url_patterns.html.path_three_asterisks_example"
+msgstr ""
+"three asterisks (e.g. <code>/mail/dmarc/***</code>) - match all paths "
+"resulting from substituting <code>***</code> with <span "
+"class=\"bold\">zero or more</span> segments (e.g. "
+"<code>/mail/dmarc/spf</code>, <code>/mail/dmarc</code> or "
+"<code>/mail/dmarc/dkim/failure</code> but <span class=\"bold\">not</span>"
+" <code>/mail/</code>)"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:92
+msgid "doc.url_patterns.html.path_trailing_slash"
+msgstr ""
+"If pattern ends <span class=\"bold\">without</span> a trailing slash, it "
+"mathes paths with any number of trailing slashes, including zero. If "
+"pattern ends <span class=\"bold\">with</span> a trailing slash, it only "
+"mathes paths with one or more  trailing slashes. For example, "
+"<code>/itscalled/*</code> matches <code>/itscalled/gnulinux</code>, "
+"<code>/itscalled/gnulinux/</code> and <code>/itscalled/gnulinux//</code> "
+"while <code>/itscalled/*/</code> only matches "
+"<code>/itscalled/gnulinux/</code> and <code>/itscalled/gnulinux//</code> "
+"out of those three."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:96
+msgid "doc.url_patterns.html.path_trailing_slash_priority"
+msgstr ""
+"If two patterns only differ by the presence of a trailing slash, pattern "
+"<span class=\"bold\">with</span> a trailing slash is considered <span "
+"class=\"bold\">more specific</span>."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:100
+msgid "doc.url_patterns.html.path_literal_trailing_asterisks"
+msgstr ""
+"Additionally, any path with literal trailing asterisks is matched by "
+"itself, even if such pattern would otherwise be treated as wildcard (e.g."
+" <code>/gobacktoxul/**</code> matches <code>/gobacktoxul/**</code>). This"
+" is likely to change in the future and would best not be relied upon. "
+"Appending three additional asterisks to path pattern to represent literal"
+" asterisks is being considered."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:105
+msgid "doc.url_patterns.label.protocol_wildcards"
+msgstr "URL scheme wildcard"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:108
+msgid "doc.url_patterns.html.protocol_wildcards"
+msgstr ""
+"<code>http://</code> and <code>https://</code> shemes in the URL are "
+"matched exactly. However, starting with Haketilo 3.0, it is also possible"
+" for scheme pseudo-wildcard of <code>http*://</code> to be used. Use of "
+"URL pattern with this scheme is equivalent to the use of 2 separate "
+"patterns starting with <code>http://</code> and <code>https://</code>, "
+"respectively. For example, pattern <code>http*://example.com</code> shall"
+" match both <code>https://example.com</code> and "
+"<code>http://example.com</code>."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:112
+msgid "doc.url_patterns.html.protocol_wildcards_are_aliases"
+msgstr ""
+"<code>http*://</code> may be considered not to be a true wildcard but "
+"rather an alias for either of the other 2 values. As of Haketilo 3.0, the"
+" speicificity of a URL pattern starting with <code>http*://</code> is "
+"considered to be the same as that of the corresponding URL pattern "
+"starting with <code>http://</code> or <code>https://</code>. In case of a"
+" conflict, the order of precedence of such patterns is unspecified. This "
+"behavior is likely to change in the future versions of Haketilo."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:117
+msgid "doc.url_patterns.label.wildcard_priorities"
+msgstr "Wildcard pattern priorities and querying"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:120
+msgid "doc.url_patterns.priorities_intro"
+msgstr ""
+"In case multiple patterns match some URL, the more specific one is "
+"preferred. Specificity is considered as follows"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:125
+msgid "doc.url_patterns.priorities_rule_path_ending"
+msgstr ""
+"If patterns only differ in the final path segment, the one with least "
+"wildcard asterisks in that segment if preferred."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:128
+msgid "doc.url_patterns.priorities_rule_path_length"
+msgstr ""
+"If patterns, besides the above, only differ in path length, one with "
+"longer path is preferred. Neither final wildcard segment nor trailing "
+"dashes account for path length."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:131
+msgid "doc.url_patterns.priorities_rule_domain_beginning"
+msgstr ""
+"If patterns, besides the above, only differ in the initial domain "
+"segment, one with least wildcard asterisks in that segment is preferred."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:134
+#, fuzzy
+msgid "doc.url_patterns.priorities_rule_domain_length"
+msgstr ""
+"If patterns differ in domain length, one with longer domain is preferred."
+" Initial wildcard segment does not account for domain length."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:139
+msgid "doc.url_patterns.html.priorities_example1_intro"
+msgstr ""
+"As an example, consider the URL "
+"<code>http://settings.query.example.com/google/tries/destroy/adblockers//</code>."
+" Patterns matching it are, in the following order"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:266
+msgid "doc.url_patterns.html.priorities_example1_note"
+msgstr ""
+"Variants of those patterns starting with <code>http*://</code> would of "
+"course match as well. They have been omitted for simplicity."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:270
+msgid "doc.url_patterns.html.priorities_example2_intro"
+msgstr ""
+"For a simpler URL like <code>https://example.com</code> the patterns "
+"would be"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:281
+msgid "doc.url_patterns.html.priorities_example2_note"
+msgstr ""
+"Variants of those patterns with a trailing dash added would <span "
+"class=\"bold\">not</span> match the URL. Also, the pattern variants "
+"starting with <code>http*://</code> have been once again omitted."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:286
+msgid "doc.url_patterns.label.limits"
+msgstr "Limits"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:289
+msgid "doc.url_patterns.limits"
+msgstr ""
+"In order to prevent some easy-to-conduct DoS attacks, older versions of "
+"Haketilo and Hydrilla limited the lengths of domain and path parts of "
+"processed URLs. This is no longer the case."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:294
+msgid "doc.url_patterns.heading.alt_solution"
+msgstr "Alternative solution idea: mimicking web server mechanics"
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:297
+msgid "doc.url_patterns.url_pattern_drawbacks"
+msgstr ""
+"While wildcard patterns as presented give a lot of flexibility, they are "
+"not the only viable approach to specifying what URLs to apply "
+"rules/payloads to. In fact, wildcards are different from how the server "
+"side of a typical website decides what to return for a given URL request."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:301
+msgid "doc.url_patterns.server_behavior_mimicking_idea"
+msgstr ""
+"In a typical scenario, an HTTP server like Apache reads configuration "
+"files provided by its administrator and uses various (virtual host, "
+"redirect, request rewrite, CGI, etc.) instructions to decide how to "
+"handle given URL. Perhps using a scheme that mimics the configuration "
+"options typically used with web servers would give more efficiency in "
+"specifying what page settings to apply when."
+
+#: src/hydrilla/proxy/self_doc/url_patterns.html.jinja:305
+msgid "doc.url_patterns.approach_may_be_considered"
+msgstr "This approach may be considered in the future."
+
 #: src/hydrilla/proxy/state_impl/concrete_state.py:111
 msgid "err.proxy.unknown_db_schema"
 msgstr ""
@@ -1449,39 +1738,39 @@ msgstr ""
 "HTTP server like Apache2 or Nginx. You can configure Hydrilla through the"
 " /etc/hydrilla/config.json file."
 
-#: src/hydrilla/url_patterns.py:139
+#: src/hydrilla/url_patterns.py:127
 msgid "err.url_pattern_{}.bad"
 msgstr "Not a valid Haketilo URL pattern: {}"
 
-#: src/hydrilla/url_patterns.py:142
+#: src/hydrilla/url_patterns.py:130
 msgid "err.url_{}.bad"
 msgstr "Not a valid URL: {}"
 
-#: src/hydrilla/url_patterns.py:149
+#: src/hydrilla/url_patterns.py:137
 msgid "err.url_pattern_{}.bad_scheme"
 msgstr "URL pattern has an unknown scheme: {}"
 
-#: src/hydrilla/url_patterns.py:152
+#: src/hydrilla/url_patterns.py:140
 msgid "err.url_{}.bad_scheme"
 msgstr "URL has an unknown scheme: {}"
 
-#: src/hydrilla/url_patterns.py:157
+#: src/hydrilla/url_patterns.py:145
 msgid "err.url_pattern_{}.special_scheme_port"
 msgstr "URL pattern has an explicit port while it shouldn't: {}"
 
-#: src/hydrilla/url_patterns.py:169
+#: src/hydrilla/url_patterns.py:157
 msgid "err.url_pattern_{}.bad_port"
 msgstr "URL pattern has a port outside of allowed range (1-65535): {}"
 
-#: src/hydrilla/url_patterns.py:172
+#: src/hydrilla/url_patterns.py:160
 msgid "err.url_{}.bad_port"
 msgstr "URL has a port outside of allowed range (1-65535): {}"
 
-#: src/hydrilla/url_patterns.py:193
+#: src/hydrilla/url_patterns.py:181
 msgid "err.url_pattern_{}.has_query"
 msgstr "URL pattern has a query string while it shouldn't: {}"
 
-#: src/hydrilla/url_patterns.py:197
+#: src/hydrilla/url_patterns.py:185
 msgid "err.url_pattern_{}.has_frag"
 msgstr "URL pattern has a fragment string while it shouldn't: {}"