initial commit

11 files changed, 1374 insertions, 0 deletions

diff --git a/src/conftest.py b/src/conftest.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/conftest.py
diff --git a/src/pydrilla/__init__.py b/src/pydrilla/__init__.py
new file mode 100644
index 0000000..8d1565b
--- /dev/null
+++ b/src/pydrilla/__init__.py
@@ -0,0 +1 @@
+from .pydrilla import create_app
diff --git a/src/pydrilla/config.json b/src/pydrilla/config.json
new file mode 100644
index 0000000..a6c4bf0
--- /dev/null
+++ b/src/pydrilla/config.json
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: CC0-1.0
+
+// Example Hydrilla config file.
+//
+// Copyright (C) 2021 Wojtek Kosior
+//
+// Available under the terms of Creative Commons Zero v1.0 Universal.
+
+{
+    "content_dir": "/var/lib/hydrilla/content",
+    "static_resource_uri": "http://localhost:8000/",
+    "try_configs": ["/etc/pydrilla/config.json"]
+}
diff --git a/src/pydrilla/development_config.json b/src/pydrilla/development_config.json
new file mode 100644
index 0000000..1660edb
--- /dev/null
+++ b/src/pydrilla/development_config.json
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: CC0-1.0
+
+// Hydrilla development config file.
+//
+// Copyright (C) 2021 Wojtek Kosior
+//
+// Available under the terms of Creative Commons Zero v1.0 Universal.
+
+// this config is meant to be used in development environment;
+// unlike config.json, it shall not be included in distribution
+{
+    "content_dir": "./example_content",
+
+    // Except files from content_dir to be served there (used to redirect
+    // clients).
+    "static_resource_uri": "http://localhost:8000/",
+
+    // Make Pydrilla error out on any warning
+    "werror": true
+
+    // With the below we can make Pydrilla look for missing content items in
+    // another instance instead of just erroring/warning.
+    // ,"hydrilla_parent": "https://api.hachette-hydrilla.org/0.2/"
+}
diff --git a/src/pydrilla/locales/en/LC_MESSAGES/pydrilla.po b/src/pydrilla/locales/en/LC_MESSAGES/pydrilla.po
new file mode 100644
index 0000000..f9e6a82
--- /dev/null
+++ b/src/pydrilla/locales/en/LC_MESSAGES/pydrilla.po
@@ -0,0 +1,127 @@
+# SPDX-License-Identifier: CC0-1.0
+
+# English localization
+#
+# This file is part of Hydrilla
+#
+# Copyright (C) 2021 Wojtek Kosior
+#
+# This file is free cultural work: you can redistribute it with or
+# without modification under the terms of the CC0 1.0 Universal License
+# as published by the Creative Commons Corporation.
+#
+# This file is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# CC0 1.0 Universal License for more details.
+
+msgid ""
+msgstr ""
+"Project-Id-Version: Hydrilla 0.2\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2021-11-13 19:03+0100\n"
+"PO-Revision-Date: 2021-11-06 08:42+0100\n"
+"Last-Translator: Wojtek Kosior <koszko@koszko.org>\n"
+"Language-Team: English\n"
+"Language: en\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#: pydrilla.py:97
+msgid "path_is_absolute_{}"
+msgstr "Provided path '{}' is absolute."
+
+#: pydrilla.py:104
+#, python-brace-format
+msgid "not_implemented_{what}_{where}"
+msgstr ""
+"Attempt to use '{what}' in '{where}' but this feature is not yet implemented."
+
+#: pydrilla.py:194
+#, python-brace-format
+msgid "uuid_mismatch_{identifier}"
+msgstr "Two different uuids were specified for item '{identifier}'."
+
+#: pydrilla.py:201
+#, python-brace-format
+msgid "version_clash_{identifier}_{version}"
+msgstr "Version '{version}' specified more than once for item '{identifier}'."
+
+#: pydrilla.py:297 pydrilla.py:309
+msgid "invalid_URL_{}"
+msgstr "Invalid URL/pattern: '{}'."
+
+#: pydrilla.py:301
+msgid "disallowed_protocol_{}"
+msgstr "Disallowed protocol: '{}'."
+
+#: pydrilla.py:391
+msgid "license_clash_{}"
+msgstr "License '{}' defined more than once."
+
+#: pydrilla.py:408
+msgid "source_name_clash_{}"
+msgstr "Source name '{}' used more than once."
+
+#: pydrilla.py:426
+#, python-format
+msgid "couldnt_load_definition_from_%s"
+msgstr "Couldn't load definition from '%s'."
+
+#: pydrilla.py:442
+#, python-format
+msgid "no_index_license_%(source)s_%(lic)s"
+msgstr "Unknown license '%(lic)s' used by index.json of '%(source)s'."
+
+#: pydrilla.py:449
+#, python-format
+msgid "no_resource_license_%(resource)s_%(ver)s_%(lic)s"
+msgstr ""
+"Unknown license '%(lic)s' used by resource '%(resource)s', version '%(ver)s'."
+
+#: pydrilla.py:451
+#, python-format
+msgid "no_mapping_license_%(mapping)s_%(ver)s_%(lic)s"
+msgstr ""
+"Unknown license '%(lic)s' used by mapping '%(mapping)s', version '%(ver)s'."
+
+#: pydrilla.py:474
+#, python-format
+msgid "no_dep_%(resource)s_%(ver)s_%(dep)s"
+msgstr ""
+"Unknown dependency '%(dep)s' of resource '%(resource)s', version '%(ver)s'."
+
+#: pydrilla.py:484
+#, python-format
+msgid "no_payload_%(mapping)s_%(ver)s_%(payload)s"
+msgstr ""
+"Unknown payload '%(payload)s' of mapping '%(mapping)s', version '%(ver)s'."
+
+#: pydrilla.py:512
+#, python-format
+msgid "couldnt_register_%(mapping)s_%(ver)s_%(pattern)s"
+msgstr ""
+"Couldn't register mapping '%(mapping)s', version '%(ver)s' (pattern "
+"'%(pattern)s')."
+
+#: pydrilla.py:566
+msgid "content_dir_path_not_dir"
+msgstr "Provided \"content_dir\" path does not name a direcotry."
+
+#: pydrilla.py:578
+#, python-format
+msgid "couldnt_load_content_from_%s"
+msgstr "Couldn't load content from '%s'."
+
+#: pydrilla.py:603
+msgid "config_key_absent_{}"
+msgstr "Config key \"{}\" not provided."
+
+#: templates/index.html:4
+msgid "hydrilla_welcome"
+msgstr "Welcome to Hydrilla!"
+
+#: templates/base.html:55 templates/base.html:61
+msgid "hydrilla"
+msgstr "Hydrilla"
diff --git a/src/pydrilla/pydrilla.py b/src/pydrilla/pydrilla.py
new file mode 100644
index 0000000..caf05a2
--- /dev/null
+++ b/src/pydrilla/pydrilla.py
@@ -0,0 +1,700 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# Main repository logic.
+#
+# This file is part of Hydrilla
+#
+# Copyright (C) 2021 Wojtek Kosior
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+#
+# I, Wojtek Kosior, thereby promise not to sue for violation of this
+# file's license. Although I request that you do not make use this code
+# in a proprietary program, I am not going to enforce this in court.
+
+from flask import Flask, Blueprint, current_app, url_for, abort, request
+from jinja2 import Environment, PackageLoader
+import re
+#from hashlib import sha256
+import os
+import pathlib
+import json
+import gettext
+import logging
+
+SCHEMA_VERSION = [0, 2]
+
+strip_comment_re = re.compile(r'''
+^ # match from the beginning of each line
+( # catch the part before '//' comment
+  (?: # this group matches either a string or a single out-of-string character
+    [^"/] |
+    "
+    (?: # this group matches any in-a-string character
+      [^"\\] |          # match any normal character
+      \\[^u] |          # match any escaped character like '\f' or '\n'
+      \\u[a-fA-F0-9]{4} # match an escape
+    )*
+    "
+  )*
+)
+# expect either end-of-line or a comment:
+# * unterminated strings will cause matching to fail
+# * bad comment (with '/' instead of '//') will be indicated by second group
+#   having length 1 instead of 2 or 0
+(//?|$)
+''', re.VERBOSE)
+
+def strip_json_comments(text):
+    processed = 0
+    stripped_text = []
+    for line in text.split('\n'):
+        match = strip_comment_re.match(line)
+
+        if match is None: # unterminated string
+            # ignore this error, let json module report it
+            stripped = line
+        elif len(match[2]) == 1:
+            raise json.JSONDecodeError('bad comment', text,
+                                       processed + len(match[1]))
+        else:
+            stripped = match[1]
+
+        stripped_text.append(stripped)
+        processed += len(line) + 1
+
+    return '\n'.join(stripped_text)
+
+here = pathlib.Path(__file__).resolve().parent
+
+bp = Blueprint('bp', __package__)
+
+def load_config(config_path):
+    config = {}
+    to_load = [config_path]
+    failures_ok = [False]
+
+    while to_load:
+        path = to_load.pop()
+        can_fail = failures_ok.pop()
+
+        try:
+            with open(config_path) as config_file:
+                new_config = json.loads(strip_json_comments(config_file.read()))
+        except Exception as e:
+            if can_fail:
+                continue
+            raise e from None
+
+        config.update(new_config)
+
+        for key, failure_ok in [('try_configs', True), ('use_configs', False)]:
+            paths = new_config.get(key, [])
+            paths.reverse()
+            to_load.extend(paths)
+            failures_ok.extend([failure_ok] * len(paths))
+
+    for key in ['try_configs', 'use_configs']:
+        if key in config:
+            config.pop(key)
+
+    return config
+
+def get_content_file_path(path):
+    if os.path.sep != '/':
+        path.replace('/', os.path.sep)
+
+    path = pathlib.Path(path)
+    if path.is_absolute():
+        raise ValueError(_('path_is_absolute_{}').format(path))
+
+    return path
+
+class MyNotImplError(NotImplementedError):
+    '''Raised when a planned but not-yet-completed feature is used.'''
+    def __init__(self, what, where):
+        super().__init__(_('not_implemented_{what}_{where}')
+                         .format(what=what, where=where))
+
+def normalize_version(ver):
+    '''
+    ver is an array of integers. Strip right-most zeroes from ver.
+
+    Returns a *new* array. Doesn't modify its argument.
+    '''
+    new_len = 0
+    for i, num in enumerate(ver):
+        if num != 0:
+            new_len = i + 1
+
+    return ver[:new_len]
+
+def parse_version(ver_str):
+    '''
+    Convert ver_str into an array representation, e.g. for ver_str="4.6.13.0"
+    return [4, 6, 13, 0].
+    '''
+    return [int(num) for num in ver_str.split('.')]
+
+def version_string(ver, rev=None):
+    '''
+    ver is an array of integers. rev is an optional integer. Produce string
+    representation of version (optionally with revision number), like:
+        1.2.3-5
+    No version normalization is performed.
+    '''
+    return '.'.join([str(n) for n in ver]) + ('' if rev is None else f'-{rev}')
+
+### pad_versions() and compare_versions() likely won't be needed
+
+# def pad_versions(ver1, ver2):
+#     '''
+#     Each of the arguments is an array of integers. If one of the arrays is
+#     shorter than the other, right-pad it with zeroes to make it the same
+#     length as the other one.
+
+#     Returns a tuple of *new* arrays. Doesn't modify its arguments.
+#     '''
+#     if len(ver1) < len(ver2):
+#         ver2, ver1 = pad_versions(ver2, ver1)
+#     else:
+#         ver2 = [*ver2, *([0] * (len(ver1) - len(ver2)))]
+#         ver1 = [*ver1]
+
+#     return ver1, ver2
+
+# def compare_versions(ver1, ver2, rev1=1, rev2=1):
+#     '''
+#     ver1 and ver2 are arrays of integers, with major version number being the
+#     first array item. If versions specified by arrays of different length need
+#     to be compared, the shorter array gets padded with zeroes on the right.
+#     This means that for example version 1.3 could be given as both [1, 3] and
+#     [1, 3, 0, 0] (aka 1.3.0.0) and either would mean the same.
+
+#     rev1 and rev2 are revision numbers. They are appended to padded ver1 and
+#     ver2 arrays respectively before comparison.
+
+#     This function returns -1, 0 or 1 when the first ver1 designates
+#     respectively a version lower than, equal to or greater than the one in
+#     ver2.
+#     '''
+#     ver1, ver2 = pad_versions(ver1, ver2)
+#     ver1.append(rev1)
+#     ver2.append(rev2)
+
+#     for n1, n2 in zip(ver1, ver2):
+#         if n1 < n2:
+#             return -1
+#         if n1 > n2:
+#             return 1
+
+#     return 0
+
+class VersionedContentItem:
+    '''Stores definitions of multiple versions of website content item.'''
+    def __init__(self):
+        self.uuid = None
+        self.identifier = None
+        self.by_version = {}
+        self.known_versions = []
+
+    def register_item(self, item):
+        if self.identifier is None:
+            self.identifier = item['identifier']
+            self.uuid = item['uuid']
+        elif self.uuid != item['uuid']:
+            raise ValueError(_('uuid_mismatch_{identifier}')
+                             .format(identifier=self.identifier))
+
+        ver = item['version']
+        ver_str = version_string(ver)
+
+        if ver_str in self.by_version:
+            raise ValueError(_('version_clash_{identifier}_{version}')
+                             .format(identifier=self.identifier,
+                                     version=ver_str))
+
+        self.by_version[ver_str] = item
+        self.known_versions.append(ver)
+
+class PatternTreeNode:
+    '''
+    "Pattern Tree" is how we refer to the data structure used for querying
+    Haketilo patterns. Those look like 'https://*.example.com/ab/***'. The goal
+    is to make it possible for given URL to quickly retrieve all known patterns
+    that match it.
+    '''
+    def __init__(self):
+        self.wildcard_matches = [None, None, None]
+        self.literal_match    = None
+        self.children         = {}
+
+    def search(self, segments):
+        '''
+        Yields all matches of this segments sequence against the tree that
+        starts at this node. Results are produces in order from greatest to
+        lowest pattern specificity.
+        '''
+        nodes = [self]
+
+        for segment in segments:
+            next_node = nodes[-1].children.get(segment)
+            if next_node is None:
+                break
+
+            nodes.append(next_node)
+
+        nsegments = len(segments)
+        cond_literal = lambda: len(nodes)     == nsegments
+        cond_wildcard = [
+            lambda: len(nodes) + 1 == nsegments and segments[-1] != '*',
+            lambda: len(nodes) + 1 <  nsegments,
+            lambda: len(nodes) + 1 != nsegments or  segments[-1] != '***'
+        ]
+
+        while nodes:
+            node = nodes.pop()
+
+            for item, condition in [(node.literal_match, cond_literal),
+                                    *zip(node.wildcard_matches, cond_wildcard)]:
+                if item is not None and condition():
+                    yield item
+
+    def add(self, segments, item_instantiator):
+        '''
+        Make item queryable through (this branch of) the Pattern Tree. If there
+        was not yet any item associated with the tree path designated by
+        segments, create a new one using item_instantiator() function. Return
+        all items matching this path (both the ones that existed and the ones
+        just created).
+        '''
+        node = self
+
+        for i, segment in enumerate(segments):
+            wildcards = node.wildcard_matches
+
+            child = node.children.get(segment) or PatternTreeNode()
+            node.children[segment] = child
+            node = child
+
+        if node.literal_match is None:
+            node.literal_match = item_instantiator()
+
+        if segment not in ('*', '**', '***'):
+            return [node.literal_match]
+
+        if wildcards[len(segment) - 1] is None:
+            wildcards[len(segment) - 1] = item_instantiator()
+
+        return [node.literal_match, wildcards[len(segment) - 1]]
+
+proto_regex  = re.compile(r'^(?P<proto>\w+)://(?P<rest>.*)$')
+user_re      = r'[^/?#@]+@' # r'(?P<user>[^/?#@]+)@' # discarded for now
+query_re     = r'\??[^#]*'  # r'\??(?P<query>[^#]*)' # discarded for now
+domain_re    = r'(?P<domain>[^/?#]+)'
+path_re      = r'(?P<path>[^?#]*)'
+http_regex   = re.compile(f'{domain_re}{path_re}{query_re}.*')
+ftp_regex    = re.compile(f'(?:{user_re})?{domain_re}{path_re}.*')
+
+class UrlError(ValueError):
+    pass
+
+class DeconstructedUrl:
+    '''Represents a deconstructed URL or URL pattern'''
+    def __init__(self, url):
+        self.url = url
+
+        match = proto_regex.match(url)
+        if not match:
+            raise UrlError(_('invalid_URL_{}').format(url))
+
+        self.proto = match.group('proto')
+        if self.proto not in ('http', 'https', 'ftp'):
+            raise UrlError(_('disallowed_protocol_{}').format(proto))
+
+        if self.proto == 'ftp':
+            match = ftp_regex.match(match.group('rest'))
+        elif self.proto in ('http', 'https'):
+            match = http_regex.match(match.group('rest'))
+
+        if not match:
+            raise UrlError(_('invalid_URL_{}').format(url))
+
+        self.domain = match.group('domain').split('.')
+        self.domain.reverse()
+        self.path = [*filter(None, match.group('path').split('/'))]
+
+class MappingItem:
+    '''
+    A mapping, together with one of its patterns, as stored in Pattern Tree.
+    '''
+    def __init__(self, pattern, mapping):
+        self.pattern = pattern
+        self.mapping = mapping
+
+    def register(self, patterns_by_proto):
+        '''
+        Make self queryable through the Pattern Tree that starts with the
+        protocols dictionary passed in the argument.
+        '''
+        deco = DeconstructedUrl(self.pattern)
+
+        domain_tree = patterns_by_proto.get(deco.proto) or PatternTreeNode()
+        patterns_by_proto[deco.proto] = domain_tree
+
+        for path_tree in domain_tree.add(deco.domain, PatternTreeNode):
+            for match_list in path_tree.add(deco.path, list):
+                match_list.append(self)
+
+class Content:
+    '''Stores serveable website content.'''
+    def __init__(self):
+        self.resources = {}
+        self.mappings  = {}
+        self.licenses  = {}
+        self.indexes   = {}
+        self.definition_processors = {
+            'resource': self.process_resource_or_mapping,
+            'mapping': self.process_resource_or_mapping,
+            'license': self.process_license
+        }
+        self.patterns_by_proto = {}
+
+    @staticmethod
+    def register_item(dict, item):
+        '''
+        Helper function used to add a versioned item definition to content
+        data structures.
+        '''
+        identifier = item['identifier']
+        versioned_item = dict.get(identifier)
+        if versioned_item is None:
+            versioned_item = VersionedContentItem()
+            dict[identifier] = versioned_item
+
+        versioned_item.register_item(item)
+
+    @staticmethod
+    def _process_copyright_and_license(definition):
+        '''Helper function used by other process_*() methods.'''
+        for field in ['copyright', 'licenses']:
+            if definition[field] == 'auto':
+                raise MyNotImplError(f'"{{field}}": "auto"',
+                                     definition['source_name'])
+
+    def process_resource_or_mapping(self, definition, index):
+        '''
+        Sanitizes, autocompletes and registers serveable mapping/resource
+        definition.
+        '''
+        definition['version'] = normalize_version(definition['version'])
+
+        if definition['type'] == 'resource':
+            self._process_copyright_and_license(definition)
+            definition['dependencies'] = definition.get('dependencies', [])
+            self.register_item(self.resources, definition)
+        else:
+            self.register_item(self.mappings, definition)
+
+    def process_license(self, license, index):
+        '''Sanitizes and registers serveable license definition.'''
+        identifier = license['identifier']
+        if identifier in self.licenses:
+            raise ValueError(_('license_clash_{}').format(identifier))
+
+        self.licenses[identifier] = license
+
+    def process_index(self, index, source_name):
+        '''
+        Sanitizes, autocompletes and registers data from a loaded index.json
+        file.
+        '''
+        schema_ver = normalize_version(index['schema_version'])
+        index['schema_version'] = schema_ver
+        if schema_ver != SCHEMA_VERSION:
+            raise ValueError('index_json_schema_mismatch_{found}_{required}'
+                             .format(found=version_string(schema_ver),
+                                     required=version_string(SCHEMA_VERSION)))
+
+        if source_name in self.indexes:
+            raise ValueError(_('source_name_clash_{}').format(source_name))
+
+        index['source_name'] = source_name
+
+        self._process_copyright_and_license(index)
+
+        self.indexes[source_name] = index
+
+        for definition in index['definitions']:
+            try:
+                definition['source_name'] = source_name
+                definition['source_copyright'] = index['copyright']
+                definition['source_licenses'] = index['licenses']
+                processor = self.definition_processors[definition['type']]
+                processor(definition, index)
+            except Exception as e:
+                if current_app._pydrilla_werror:
+                    raise e from None
+                logging.error(_('couldnt_load_definition_from_%s'), subdir_path,
+                              exc_info=True)
+    @staticmethod
+    def all_items(versioned_items_dict):
+        '''Iterator over all registered versions of all items.'''
+        for versioned_item in versioned_items_dict.values():
+            for item in versioned_item.by_version.values():
+                yield item
+
+    def report_missing(self):
+        '''
+        Use logger to print information about items that are referenced but
+        were not loaded.
+        '''
+        def report_missing_license(object, object_type, lic):
+            if object_type == 'index':
+                logging.error(_('no_index_license_%(source)s_%(lic)s'),
+                              source=object['source_name'], lic=lic)
+                return
+
+            ver_str = version_string(object['version'])
+            kwargs = {object_type: object['identifier'], ver: ver_str, lic: lic}
+            if object_type == 'resource':
+                fmt = _('no_resource_license_%(resource)s_%(ver)s_%(lic)s')
+            else:
+                fmt = _('no_mapping_license_%(mapping)s_%(ver)s_%(lic)s')
+
+            logging.error(fmt, **kwargs)
+
+        for object_type, iterable in [
+                ('index',    self.indexes.values()),
+                ('resource', self.all_items(self.resources))
+        ]:
+            for object in iterable:
+                to_process = [object['licenses']]
+                licenses = []
+                while to_process:
+                    term = to_process.pop()
+
+                    if type(term) is str:
+                        if term not in ['or', 'and'] and \
+                           term not in self.licenses:
+                            report_missing_license(object, object_type, lic)
+                        continue
+
+                    to_process.extend(term)
+
+        def report_missing_dependency(resource, dep):
+            logging.error(_('no_dep_%(resource)s_%(ver)s_%(dep)s'),
+                          dep=dep, resource=resource['identifier'],
+                          ver=version_string(resource['version']))
+
+        for resource in self.all_items(self.resources):
+            for dep in resource['dependencies']:
+                if dep not in self.resources:
+                    report_missing_dependency(resource, dep)
+
+        def report_missing_payload(mapping, payload):
+            logging.error(_('no_payload_%(mapping)s_%(ver)s_%(payload)s'),
+                          mapping=mapping['identifier'], payload=payload,
+                          ver=version_string(mapping['version']))
+
+        for mapping in self.all_items(self.mappings):
+            for payload in mapping['payloads']:
+                payload = payload['payload']
+                if payload not in self.resources:
+                    report_missing_payload(mapping, payload)
+
+    def finalize(self):
+        '''
+        Initialize structures needed to serve queries. Called once after all
+        data gets loaded.
+        '''
+        for dict in [self.resources, self.mappings]:
+            for versioned_item in dict.values():
+                versioned_item.known_versions.sort()
+
+        for mapping in self.all_items(self.mappings):
+            for payload in mapping['payloads']:
+                try:
+                    MappingItem(pattern, mapping)\
+                        .register(self.patterns_by_proto)
+                except Exception as e:
+                    if current_app._pydrilla_werror:
+                        raise e from None
+                    logging.error(
+                        _('couldnt_register_%(mapping)s_%(ver)s_%(pattern)s'),
+                        mapping=mapping['identifier'], pattern=pattern,
+                        ver=version_string(mapping['version'])
+                    )
+
+    def find_item(self, type, identifier, ver=None):
+        '''
+        Find and return definition of the newest version of resource/mapping
+        named by identifier. If no such resource/mapping exists, return None.
+
+        If ver is specified, instead find and return definition of that version
+        of the item (or None is absent).
+        '''
+        dict = self.resources if type == 'resource' else self.mappings
+        versioned_item = dict.get(identifier)
+        if not versioned_item:
+            return None
+
+        ver = version_string(ver or versioned_item.known_versions[-1])
+
+        return versioned_item.by_version.get(ver)
+
+    def query(self, url, max=0):
+        '''
+        Return return registered patterns and mappings (available as
+        MappingItems) that match url. The maximum number of items yielded may be
+        limited by using the optional max argument. Its default value, 0, causes
+        no limit to be imposed.
+
+        If multiple versions of a mapping are applicable, only the most recent
+        is included in the result.
+        '''
+        deco = DeconstructedUrl(url)
+
+        domain_tree = self.patterns_by_proto.get(deco.proto) \
+            or PatternTreeNode()
+        for path_tree in domain_tree.search(deco.domain):
+            for item in path_tree.search(deco.path):
+                if url[-1] == '/' or item.pattern[-1] != '/':
+                    yield item
+                    max -= 1
+                    if max == 0:
+                        return
+
+def load_content_from_subdir(subdir_path, source_name, content):
+    index_path = subdir_path / 'index.json'
+    with open(index_path) as index_file:
+        index = json.loads(strip_json_comments(index_file.read()))
+
+    content.process_index(index, source_name)
+
+def load_content(path):
+    path = pathlib.Path(path)
+    if not path.is_dir():
+        raise ValueError(_('content_dir_path_not_dir'))
+
+    content = Content()
+
+    for subdir_path in path.iterdir():
+        if not subdir_path.is_dir():
+            continue
+        try:
+            load_content_from_subdir(subdir_path, subdir_path.name, content)
+        except Exception as e:
+            if current_app._pydrilla_werror:
+                raise e from None
+            logging.error(_('couldnt_load_content_from_%s'), subdir_path,
+                          exc_info=True)
+
+    content.report_missing()
+    content.finalize()
+
+    return content
+
+def create_app(config_path=(here / 'config.json'), flask_config={}):
+    app = Flask(__package__)
+    app.config.update(flask_config)
+
+    language = flask_config.get('lang', 'en')
+    translation = gettext.translation('pydrilla', localedir=(here / 'locales'),
+                                      languages=[language])
+
+    app._pydrilla_gettext = translation.gettext
+
+    # https://stackoverflow.com/questions/9449101/how-to-stop-flask-from-initialising-twice-in-debug-mode
+    if app.debug and os.environ.get('WERKZEUG_RUN_MAIN') != 'true':
+        return app
+
+    config = load_config(config_path)
+    for key in ['static_resource_uri', 'content_dir']:
+        if key not in config:
+            raise ValueError(_('config_key_absent_{}').format(key))
+
+    app._pydrilla_static_resource_uri = config['static_resource_uri']
+    app._pydrilla_werror = config.get('werror', False)
+    if 'hydrilla_parent' in config:
+        raise MyNotImplError('hydrilla_parent', config_path.name)
+    with app.app_context():
+        app._pydrilla_content = load_content(config['content_dir'])
+
+    app.register_blueprint(bp)
+
+    return app
+
+def _(text_key):
+    return current_app._pydrilla_gettext(text_key)
+
+def escaping_gettext(text_key):
+    from markupsafe import escape
+
+    return str(escape(_(text_key)))
+
+class MyEnvironment(Environment):
+    '''
+    A wrapper class around jinja2.Environment that causes GNU gettext function
+    (as '_' and '__') and url_for function to be passed to every call of each
+    template's render() method.
+    '''
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def get_template(self, *args, **kwargs):
+        template = super().get_template(*args, **kwargs)
+        old_render = template.render
+
+        def new_render(*args, **kwargs):
+            final_kwargs = {
+                '_': escaping_gettext,
+                '__': escaping_gettext,
+                'url_for': url_for
+            }
+            final_kwargs.update(kwargs)
+
+            return old_render(*args, **final_kwargs)
+
+        template.render = new_render
+
+        return template
+
+j2env = MyEnvironment(loader=PackageLoader(__package__), autoescape=False)
+
+indexpage = j2env.get_template('index.html')
+@bp.route('/')
+def index():
+    return indexpage.render(content=current_app._pydrilla_resources_map)
+
+for item_type in ['resource', 'mapping']:
+    def item(identifier):
+        ver = request.args.get('ver')
+        if ver is not None:
+            try:
+                ver = normalize_version(parse_version(ver))
+            except:
+                abort(400)
+
+        item = current_app._pydrilla_content\
+                          .find_item(item_type, identifier, ver)
+        if item is None:
+            abort(404)
+
+        return json.dumps(item)
+
+    item.__name__ = item_type + 's'
+    bp.route(f'/{item_type}s/<string:identifier>')(item)
diff --git a/src/pydrilla/templates/base.html b/src/pydrilla/templates/base.html
new file mode 100644
index 0000000..6e7887e
--- /dev/null
+++ b/src/pydrilla/templates/base.html
@@ -0,0 +1,94 @@
+{# SPDX-License-Identifier: CC-BY-NC-SA-4.0
+
+Base HTML page template.
+
+This file is part of Hydrilla
+
+Copyright (C) 2021 Wojtek Kosior
+
+This file is free cultural work: you can redistribute it with or
+without modification under the terms of the Creative Commons
+Attribution Share Alike 4.0 International as published by the
+Creative Commons Corporation.
+
+This file is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Creative Commons Attribution Share Alike 4.0 International
+License for more details.
+
+
+I, Wojtek Kosior, thereby promise not to sue for violation of this
+file's license. Although I request that you do not make use this code
+in a proprietary program, I am not going to enforce this in court.
+#}
+
+{% macro link_for(endpoint, text) -%}
+  <a href="{{ url_for(endpoint, **kwargs) }}"
+     {{ caller() if caller is defined }}>
+     {{ text }}
+  </a>
+{%- endmacro %}
+
+<!DOCTYPE html>
+<html>
+  <head>
+    {% block head %}
+      <meta http-equiv="Content-Security-Policy" content="script-src 'none';">
+      <style>
+	{% block styles %}
+	  html, body, div, h1, h2, h3, h4, h5, h6 {
+	    margin: 0;
+	    padding: 0;
+	  }
+
+	  * {
+	    color: #444;
+	  }
+
+	  aside {
+	    display: inline-block;
+            border-left: 0.2em solid #e44;
+            background-color: #edc;
+            padding: 0.2em;
+	  }
+
+	  .nav {
+	    background-color: #ddd;
+	  }
+
+	  .nav>*:hover {
+	    background-color: #999;
+	  }
+
+	  .nav>* {
+	    display: inline-block;
+	    padding: 1em;
+	  }
+
+	  .nav a {
+ 	    text-decoration: none;
+	  }
+
+	  .home_link {
+  	    font-weight: bold;
+	    font-size: 1.5em;
+	    padding: 0.5em;
+	  }
+	{% endblock %}
+      </style>
+      <title>{% block title %}{{ _('hydrilla') }}{% endblock %}</title>
+    {% endblock %}
+  </head>
+  <body>
+    {% block body %}
+      <div class="nav">
+	{% call link_for('bp.index', _('hydrilla')) %}
+	  class="home_link"
+	{% endcall %}
+      </div>
+      {% block content %}
+      {% endblock %}
+    {% endblock %}
+  </body>
+</html>
diff --git a/src/pydrilla/templates/index.html b/src/pydrilla/templates/index.html
new file mode 100644
index 0000000..71de8ba
--- /dev/null
+++ b/src/pydrilla/templates/index.html
@@ -0,0 +1,32 @@
+{# SPDX-License-Identifier: CC-BY-NC-SA-4.0
+
+HTML index page template.
+
+This file is part of Hydrilla
+
+Copyright (C) 2021 Wojtek Kosior
+
+This file is free cultural work: you can redistribute it with or
+without modification under the terms of the Creative Commons
+Attribution Share Alike 4.0 International as published by the
+Creative Commons Corporation.
+
+This file is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Creative Commons Attribution Share Alike 4.0 International
+License for more details.
+
+
+I, Wojtek Kosior, thereby promise not to sue for violation of this
+file's license. Although I request that you do not make use this code
+in a proprietary program, I am not going to enforce this in court.
+#}
+
+{% extends 'base.html' %}
+{% block body %}
+  {{ super() }}
+  <h2>{{ _('hydrilla_welcome') }}</h2>
+  <h4>content</h4>
+  {{ content }}
+{% endblock %}
diff --git a/src/pydrilla_dev_helper.py b/src/pydrilla_dev_helper.py
new file mode 100644
index 0000000..88dc63e
--- /dev/null
+++ b/src/pydrilla_dev_helper.py
@@ -0,0 +1,293 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# Definitions of helper commands to use with setuptools
+#
+# This file is part of Hydrilla
+#
+# Copyright (C) 2021 Wojtek Kosior
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+#
+# I, Wojtek Kosior, thereby promise not to sue for violation of this file's
+# license. Although I request that you do not make use this code in a
+# proprietary program, I am not going to enforce this in court.
+
+from setuptools import Command
+from setuptools.command.build_py import build_py
+import sys
+from pathlib import Path
+import subprocess
+import re
+import os
+import json
+import importlib
+
+def mypath(path_or_string):
+    return Path(path_or_string).resolve()
+
+debrel_regex = re.compile(r'^[^(]*\([^-]*-([^)]*)\)')
+
+def extract_debrel(debian_dir):
+    changelog_path = mypath(debian_dir) / 'changelog'
+    with open(changelog_path) as changelog_file:
+        try:
+            return debrel_regex.match(changelog_file.readline())[1]
+        except TypeError:
+            raise RuntimeException('Cannot extract debrel from %s.' %
+                                   changelog_path)
+
+class Helper:
+    def __init__(self, project_root, app_package_name, version, locales_dir,
+                 locales=['en', 'pl'], default_locale='en', locale_domain=None,
+                 packages_root=None, debian_dir=None, config_path=None):
+        self.project_root       = mypath(project_root)
+        self.app_package_name   = app_package_name
+        self.version            = version
+        self.locales_dir        = mypath(locales_dir)
+        self.locales            = locales
+        self.default_locale     = default_locale
+        self.locale_domain      = locale_domain or app_package_name
+        self.packages_root      = mypath(packages_root or project_root / 'src')
+        self.app_package_dir    = self.packages_root / app_package_name
+        self.debian_dir         = mypath(debian_dir or project_root / 'debian')
+        self.config_path        = config_path and mypath(config_path)
+        self.locale_files_list  = None
+
+    def run_command(self, command, verbose, runner=subprocess.run, **kwargs):
+        cwd = kwargs.get('cwd')
+        if cwd:
+            cwd = mypath(cwd)
+            where = f'from {cwd} '
+        else:
+            cwd = Path.cwd().resolve()
+            where = ''
+
+        str_command = [str(command[0])]
+
+        for arg in command[1:]:
+            if isinstance(arg, Path):
+                try:
+                    arg = str(arg.relative_to(cwd))
+                except ValueError:
+                    arg = str(arg)
+
+            str_command.append(arg)
+
+        if verbose:
+            print(f'{where}executing {" ".join(str_command)}')
+        runner(str_command, **kwargs)
+
+    def create_mo_files(self, dry_run=False, verbose=False):
+        self.locale_files_list = []
+
+        for locale in self.locales:
+            messages_dir = self.locales_dir / locale / 'LC_MESSAGES'
+
+            for po_path in messages_dir.glob('*.po'):
+                mo_path = po_path.with_suffix('.mo')
+
+                if not dry_run:
+                    command = ['msgfmt', po_path, '-o', mo_path]
+                    self.run_command(command, verbose=verbose, check=True)
+
+            self.locale_files_list.extend([po_path, mo_path])
+
+    def locale_files(self):
+        if self.locale_files_list is None:
+            self.create_mo_files(dry_run=True)
+
+        return self.locale_files_list
+
+    def locale_files_relative(self, to=None):
+        if to is None:
+            to = self.app_package_dir
+
+        return [file.relative_to(to) for file in self.locale_files()]
+
+    def flask_run(self, locale=None):
+        for var, val in (('ENV', 'development'), ('DEBUG', 'True')):
+            os.environ[f'FLASK_{var}'] = os.environ.get(f'FLASK_{var}', val)
+
+        config = {'lang': locale or self.default_locale}
+
+        sys.path.insert(0, str(self.packages_root))
+        package = importlib.import_module(self.app_package_name)
+
+        # make relative paths in json config resolve from project's directory
+        os.chdir(self.project_root)
+
+        kwargs = {'config_path': self.config_path} if self.config_path else {}
+        package.create_app(flask_config=config, **kwargs).run()
+
+    def update_po_files(self, verbose=False):
+        pot_path = self.locales_dir / f'{self.locale_domain}.pot'
+        rglob = self.app_package_dir.rglob
+        command = ['xgettext', '-d', self.locale_domain, '--language=Python',
+                   '-o', pot_path, *rglob('*.py'), *rglob('*.html')]
+
+        self.run_command(command, verbose=verbose, check=True,
+                         cwd=self.app_package_dir)
+
+        for locale in self.locales:
+            messages_dir = self.locales_dir / locale / 'LC_MESSAGES'
+
+            for po_path in messages_dir.glob('*.po'):
+                if po_path.stem != self.app_package_name:
+                    continue;
+
+                if po_path.exists():
+                    command = ['msgmerge', '--update', po_path, pot_path]
+                else:
+                    command = ['cp', po_path, pot_path]
+
+                self.run_command(command, verbose=verbose, check=True)
+
+        if (verbose):
+            print('removing generated .pot file')
+        pot_path.unlink()
+
+    # we exclude these from the source archive we produce
+    bad_file_regex = re.compile(r'^\..*|build|debian|dist')
+
+    def make_tarballs(self, verbose=False):
+        name=self.app_package_name
+        ver=self.version
+        debrel=extract_debrel(self.debian_dir)
+
+        source_dirname      = f'{name}-{ver}'
+        source_tarball_name = f'{name}_{ver}.orig.tar.gz'
+        debian_tarball_name = f'{name}_{ver}-{debrel}.debian.tar.gz'
+
+        source_args = [f'--prefix={source_dirname}/', '-o',
+                       self.project_root.parent / source_tarball_name, 'HEAD']
+
+        for filepath in self.project_root.iterdir():
+            if not self.bad_file_regex.search(filepath.parts[-1]):
+                 source_args.append(filepath)
+
+        debian_args = ['-o', self.project_root.parent / debian_tarball_name,
+                       'HEAD', self.debian_dir]
+
+        for args in [source_args, debian_args]:
+            command = ['git', 'archive', '--format=tar.gz', *args]
+            self.run_command(command, verbose=verbose, check=True)
+
+    def commands(self):
+        helper = self
+
+        class MsgfmtCommand(Command):
+            '''A custom command to run msgfmt on all .po files below '{}'.'''
+
+            description = 'use msgfmt to generate .mo files from .po files'
+            user_options = []
+
+            def initialize_options(self):
+                pass
+
+            def finalize_options(self):
+                pass
+
+            def run(self):
+                helper.create_mo_files(verbose=self.verbose)
+
+        MsgfmtCommand.__doc__ = MsgfmtCommand.__doc__.format(helper.locales_dir)
+
+        class RunCommand(Command):
+            '''
+            A custom command to run the app using flask.
+
+            This is similar in effect to:
+                PYTHONPATH='{packages_root}' FLASK_APP={app_package_name} \\
+                    FLASK_ENV=development flask run
+            '''
+
+            description = 'run the Flask app from source directory'
+
+            user_options = [
+                ('locale=', 'l',
+                 "app locale (one of: %s; default: '%s')" %
+                 (', '.join([f"'{l}'" for l in helper.locales]),
+                  helper.default_locale))
+            ]
+
+            def initialize_options(self):
+                self.locale = helper.default_locale
+
+            def finalize_options(self):
+                if self.locale not in helper.locales:
+                    raise ValueError("Locale '%s' not supported" % self.lang)
+
+            def run(self):
+                helper.flask_run(locale=self.locale)
+
+        RunCommand.__doc__ = RunCommand.__doc__.format(
+            packages_root=self.packages_root,
+            app_package_name=self.app_package_name
+        )
+
+        class MsgmergeCommand(Command):
+            '''
+            A custom command to run xgettext and msgmerge to update project's
+            .po files below '{}'.
+            '''
+
+            description = 'use xgettext and msgmerge to update (or generate) .po files for this project'
+            user_options = []
+
+            def initialize_options(self):
+                pass
+
+            def finalize_options(self):
+                pass
+
+            def run(self):
+                helper.update_po_files(verbose=self.verbose)
+
+        MsgmergeCommand.__doc__ = \
+            MsgmergeCommand.__doc__.format(helper.locales_dir)
+
+        class TarballsCommand(Command):
+            '''
+            A custom command to run git archive to create debian tarballs of
+            this project.
+            '''
+
+            description = 'use git archive to create .orig.tar.gz and .debian.tar.gz files for this project'
+            user_options = []
+
+            def initialize_options(self):
+                pass
+
+            def finalize_options(self):
+                pass
+
+            def run(self):
+                helper.make_tarballs(verbose=self.verbose)
+
+        class BuildCommand(build_py):
+            '''
+            The build command but runs the custom msgfmt command before build.
+            '''
+            def run(self, *args, **kwargs):
+                self.run_command('msgfmt')
+                super().run(*args, **kwargs)
+
+        return {
+            'msgfmt': MsgfmtCommand,
+            'run': RunCommand,
+            'msgmerge': MsgmergeCommand,
+            'tarballs': TarballsCommand,
+            'build_py': BuildCommand
+        }
diff --git a/src/test/__init__.py b/src/test/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/test/__init__.py
diff --git a/src/test/test_pydrilla.py b/src/test/test_pydrilla.py
new file mode 100644
index 0000000..0ed5fa9
--- /dev/null
+++ b/src/test/test_pydrilla.py
@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# Repository tests
+#
+# This file is part of Hydrilla
+#
+# Copyright (C) 2021 Wojtek Kosior
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+#
+# I, Wojtek Kosior, thereby promise not to sue for violation of this
+# file's license. Although I request that you do not make use this code
+# in a proprietary program, I am not going to enforce this in court.
+
+import pytest
+import sys
+import shutil
+from pathlib import Path
+from os import mkdir, unlink, environ
+import json
+
+import pydrilla
+
+test_dir     = Path(__file__).resolve().parent
+pydrilla_dir = Path(hydrilla.__file__).resolve().parent
+test_config_path = test_dir / 'config.json'
+
+@pytest.fixture
+def client():
+    app = pydrilla.create_app(test_config_path, flask_config={'TESTING': True})
+
+    with app.test_client() as client:
+        yield client
+
+def test_basic(client):
+    response = client.get('/')
+    assert b'html' in response.data
+
+def test_normalize_version():
+    assert pydrilla.normalize_version([4, 5, 3, 0, 0]) == [4, 5, 3]
+    assert pydrilla.normalize_version([1, 0, 5, 0])    == [1, 0, 5]
+    assert pydrilla.normalize_version([3, 3])          == [3, 3]
+
+### pad_versions() and compare_versions() likely won't be needed
+
+# def test_compare_versions():
+#     compare_versions = pydrilla.compare_versions
+#     # without revision
+#     assert compare_versions([43],       [43])       == 0
+#     assert compare_versions([54],       [34])       == 1
+#     assert compare_versions([1],        [3])        == -1
+#     assert compare_versions([10, 2],    [10, 2])    == 0
+#     assert compare_versions([11, 6],    [11, 2])    == 1
+#     assert compare_versions([3,  0],    [3,  8])    == -1
+#     assert compare_versions([1,  2, 3], [1,  2])    == 1
+#     assert compare_versions([1,  2],    [1,  2, 3]) == -1
+#     assert compare_versions([1],        [1,  0, 0]) == 0
+
+#     # with revision
+#     assert compare_versions([43],       [43],        rev2=3)          == -1
+#     assert compare_versions([54],       [34]),       rev2=41)         == 1
+#     assert compare_versions([1],        [3]),        rev1=6)          == -1
+#     assert compare_versions([10, 2],    [10, 2]),    rev1=8, rev2=5)  == 1
+#     assert compare_versions([11, 6],    [11, 2]),    rev2=19)         == 1
+#     assert compare_versions([3,  0],    [3,  8]),    rev2=5)          == -1
+#     assert compare_versions([1,  2, 3], [1,  2]),    rev1=4)          == 1
+#     assert compare_versions([1,  2],    [1,  2, 3]), rev2=7)          == -1
+#     assert compare_versions([1],        [1,  0, 0]), rev2=9, rev1=9)  == 0
+
+#     from functools import cmp_to_key
+
+#     versions = [[43], [54], [3, 0], [34], [3], [1], [4, 5, 3], [1, 0, 5],
+#                 [3, 3], [10, 2], [11, 2], [11, 6], [3, 8], [1, 2], [1, 2, 3],
+#                 [1, 0, 0]]
+#     versions.sort(cmp_to_key(compare_versions))
+#     assert versions == [[1], [1, 0, 0], [1, 0, 5], [1, 2], [1, 2, 3], [3, 0],
+#                         [3], [3, 3], [3, 8], [4, 5, 3], [10, 2], [11, 2],
+#                         [11, 6], [34], [43], [54]]