bring Hydrilla server part back to a usable state

1 files changed, 72 insertions, 232 deletions

diff --git a/src/hydrilla/server/serve.py b/src/hydrilla/server/serve.py
index 8f0d557..823437a 100644
--- a/src/hydrilla/server/serve.py
+++ b/src/hydrilla/server/serve.py
@@ -29,221 +29,29 @@ from __future__ import annotations
 
 import re
 import os
-import pathlib
 import json
-import logging
+import typing as t
 
 from pathlib import Path
-from hashlib import sha256
-from abc import ABC, abstractmethod
-from typing import Optional, Union, Iterable, TypeVar, Generic
 
 import click
 import flask
+import werkzeug
 
-from werkzeug import Response
-
-from .. import _version, versions, json_instances
-from ..item_infos import ResourceInfo, MappingInfo, VersionedItemInfo
+from ..exceptions import HaketiloException
+from .. import _version
 from ..translations import smart_gettext as _, translation as make_translation
-#from ..url_patterns import PatternTree
+from .. import versions
+from .. import item_infos
 from . import config
+from . import malcontent
 
-here = Path(__file__).resolve().parent
 
 generated_by = {
     'name': 'hydrilla.server',
     'version': _version.version
 }
 
-    # def as_query_result(self) -> dict[str, Union[str, list[int]]]:
-    #     """
-    #     Produce a json.dump()-able object describing this mapping as one of a
-    #     collection of query results.
-    #     """
-    #     return {
-    #         'version':    self.version,
-    #         'identifier': self.identifier,
-    #         'long_name':  self.long_name
-    #     }
-
-class Malcontent:
-    """
-    Represent a directory with files that can be loaded and served by Hydrilla.
-    """
-    def __init__(self, malcontent_dir_path: Path):
-        """
-        When an instance of Malcontent is constructed, it searches
-        malcontent_dir_path for serveable site-modifying packages and loads
-        them into its data structures.
-        """
-        self.resource_infos: dict[str, VersionedItemInfo[ResourceInfo]] = {}
-        self.mapping_infos:  dict[str, VersionedItemInfo[MappingInfo]]  = {}
-
-        self.pattern_tree: PatternTree[MappingInfo] = PatternTree()
-
-        self.malcontent_dir_path = malcontent_dir_path
-
-        if not self.malcontent_dir_path.is_dir():
-            raise ValueError(_('malcontent_dir_path_not_dir_{}')
-                             .format(malcontent_dir_path))
-
-        for item_type in ('mapping', 'resource'):
-            type_path = self.malcontent_dir_path / item_type
-            if not type_path.is_dir():
-                continue
-
-            for subpath in type_path.iterdir():
-                if not subpath.is_dir():
-                    continue
-
-                for ver_file in subpath.iterdir():
-                    try:
-                        self._load_item(item_type, ver_file)
-                    except Exception as e:
-                        if flask.current_app._hydrilla_werror:
-                            raise e from None
-
-                        msg = _('couldnt_load_item_from_{}').format(ver_file)
-                        logging.error(msg, exc_info=True)
-
-        self._report_missing()
-        self._finalize()
-
-    @staticmethod
-    def _register_info(infos: dict[str, VersionedItemInfo[VersionedType]],
-                       identifier: str, item_info: VersionedType) -> None:
-        """
-        ...........
-        """
-        infos.setdefault(identifier, VersionedItemInfo())\
-             .register(item_info)
-
-    def _load_item(self, item_type: str, ver_file: Path) -> None:
-        """
-        Reads, validates and autocompletes serveable mapping/resource
-        definition, then registers information from it in data structures.
-        """
-        version    = versions.parse_version(ver_file.name)
-        identifier = ver_file.parent.name
-
-        item_json, major = util.load_instance_from_file(ver_file)
-
-        util.validator_for(f'api_{item_type}_description-{major}.schema.json')\
-            .validate(item_json)
-
-        # Assertion needed for mypy. If validation passed, this should not fail.
-        assert major is not None
-
-        item_info: ItemInfo = ResourceInfo(item_json, major) \
-            if item_type == 'resource' else MappingInfo(item_json, major)
-
-        if item_info.identifier != identifier:
-            msg = _('item_{item}_in_file_{file}')\
-                .format({'item': item_info.identifier, 'file': ver_file})
-            raise ValueError(msg)
-
-        if item_info.version != version:
-            ver_str = util.version_string(item_info.version)
-            msg = _('item_version_{ver}_in_file_{file}')\
-                .format({'ver': ver_str, 'file': ver_file})
-            raise ValueError(msg)
-
-        if isinstance(item_info, ResourceInfo):
-            self._register_info(self.resource_infos, identifier, item_info)
-        elif isinstance(item_info, MappingInfo):
-            self._register_info(self.mapping_infos, identifier, item_info)
-
-    @staticmethod
-    def _all_infos(infos: dict[str, VersionedItemInfo[VersionedType]]) \
-        -> Iterable[VersionedType]:
-        """
-        ...........
-        """
-        for versioned_info in infos.values():
-            for item_info in versioned_info.by_version.values():
-                yield item_info
-
-    def _report_missing(self) -> None:
-        """
-        Use logger to print information about items that are referenced but
-        were not loaded.
-        """
-        def report_missing_dependency(info: ResourceInfo, dep: str) -> None:
-            msg = _('no_dep_{resource}_{ver}_{dep}')\
-                .format(dep=dep, resource=info.identifier,
-                        ver=util.version_string(info.version))
-            logging.error(msg)
-
-        for resource_info in self._all_infos(self.resource_infos):
-            for dep in resource_info.dependencies:
-                if dep not in self.resource_infos:
-                    report_missing_dependency(resource_info, dep)
-
-        def report_missing_payload(info: MappingInfo, payload: str) -> None:
-            msg = _('no_payload_{mapping}_{ver}_{payload}')\
-                .format(mapping=info.identifier, payload=payload,
-                        ver=util.version_string(info.version))
-            logging.error(msg)
-
-        for mapping_info in self._all_infos(self.mapping_infos):
-            for payload in mapping_info.payloads.values():
-                if payload not in self.resource_infos:
-                    report_missing_payload(mapping_info, payload)
-
-        def report_missing_mapping(info: ItemInfo,
-                                   required_mapping: str) -> None:
-            msg = _('no_mapping_{required_by}_{ver}_{required}')\
-                .format(required_by=info.identifier, required=required_mapping,
-                        ver=util.version_string(info.version))
-            logging.error(msg)
-
-        for item_info in (*self._all_infos(self.mapping_infos),
-                          *self._all_infos(self.resource_infos)):
-            for required in item_info.required_mappings:
-                if required not in self.mapping_infos:
-                    report_missing_mapping(item_info, required)
-
-    def _finalize(self):
-        """
-        Initialize structures needed to serve queries. Called once after all
-        data gets loaded.
-        """
-        for versioned_info in (*self.mapping_infos.values(),
-                               *self.resource_infos.values()):
-                versioned_info.known_versions.sort()
-
-        for info in self._all_infos(self.mapping_infos):
-            for pattern in info.payloads:
-                try:
-                    self.pattern_tree = \
-                        self.pattern_tree.register(pattern, info)
-                except Exception as e:
-                    if flask.current_app._hydrilla_werror:
-                        raise e from None
-                    msg = _('couldnt_register_{mapping}_{ver}_{pattern}')\
-                        .format(mapping=info.identifier, pattern=pattern,
-                                ver=util.version_string(info.version))
-                    logging.error(msg)
-
-    def query(self, url: str) -> list[MappingInfo]:
-        """
-        Return a list of registered mappings that match url.
-
-        If multiple versions of a mapping are applicable, only the most recent
-        is included in the result.
-        """
-        collected: dict[str, MappingInfo] = {}
-        for result_set in self.pattern_tree.search(url):
-            for wrapped_mapping_info in result_set:
-                info = wrapped_mapping_info.item
-                previous = collected.get(info.identifier)
-                if previous and previous.version > info.version:
-                    continue
-
-                collected[info.identifier] = info
-
-        return list(collected.values())
 
 bp = flask.Blueprint('bp', __package__)
 
@@ -267,41 +75,36 @@ class HydrillaApp(flask.Flask):
             ]
         }
 
-        self._hydrilla_project_url = hydrilla_config['hydrilla_project_url']
         self._hydrilla_port = hydrilla_config['port']
         self._hydrilla_werror = hydrilla_config.get('werror', False)
+        verify_files = hydrilla_config.get('verify_files', True)
 
         if 'hydrilla_parent' in hydrilla_config:
-            raise ValueError("Option 'hydrilla_parent' is not implemented.")
+            raise HaketiloException(_('err.server.opt_hydrilla_parent_not_implemented'))
 
-        malcontent_dir = Path(hydrilla_config['malcontent_dir']).resolve()
-        with self.app_context():
-            self._hydrilla_malcontent = Malcontent(malcontent_dir)
+        malcontent_dir_path = Path(hydrilla_config['malcontent_dir']).resolve()
+        self._hydrilla_malcontent = malcontent.Malcontent(
+            malcontent_dir_path = malcontent_dir_path,
+            werror              = self._hydrilla_werror,
+            verify_files        = verify_files
+        )
 
-        self.register_blueprint(bp)
+        self.jinja_env.install_gettext_translations(make_translation())
 
-    def create_jinja_environment(self, *args, **kwargs) \
-        -> flask.templating.Environment:
-        """
-        Flask's create_jinja_environment(), but tweaked to always include the
-        'hydrilla_project_url' global variable and to install proper
-        translations.
-        """
-        env = super().create_jinja_environment(*args, **kwargs) # type: ignore
-        env.install_gettext_translations(make_translation())
-        env.globals['hydrilla_project_url'] = self._hydrilla_project_url
+        self.jinja_env.globals['hydrilla_project_url'] = \
+            hydrilla_config['hydrilla_project_url']
 
-        return env
+        self.register_blueprint(bp)
 
     def run(self, *args, **kwargs):
         """
-        Flask's run(), but tweaked to use the port from hydrilla configuration
-        by default.
+        Flask's run() but tweaked to use the port from hydrilla configuration by
+        default.
         """
         return super().run(*args, port=self._hydrilla_port, **kwargs)
 
-def malcontent():
-    return flask.current_app._hydrilla_malcontent
+def get_malcontent() -> malcontent.Malcontent:
+    return t.cast(HydrillaApp, flask.current_app)._hydrilla_malcontent
 
 @bp.route('/')
 def index():
@@ -309,7 +112,8 @@ def index():
 
 identifier_json_re = re.compile(r'^([-0-9a-z.]+)\.json$')
 
-def get_resource_or_mapping(item_type: str, identifier: str) -> Response:
+def get_resource_or_mapping(item_type: str, identifier: str) \
+    -> werkzeug.Response:
     """
     Strip '.json' from 'identifier', look the item up and send its JSON
     description.
@@ -320,41 +124,73 @@ def get_resource_or_mapping(item_type: str, identifier: str) -> Response:
 
     identifier = match.group(1)
 
+    infos: t.Mapping[str, malcontent.VersionedItemInfo]
     if item_type == 'resource':
-        infos = malcontent().resource_infos
+        infos = get_malcontent().resource_infos
     else:
-        infos = malcontent().mapping_infos
+        infos = get_malcontent().mapping_infos
 
     versioned_info = infos.get(identifier)
 
-    info = versioned_info and versioned_info.get_by_ver()
-    if info is None:
+    if versioned_info is None:
         flask.abort(404)
 
+    info = versioned_info.newest_info()
+
     # no need for send_from_directory(); path is safe, constructed by us
-    file_path = malcontent().malcontent_dir_path / item_type / info.path()
+    info_path = f'{info.identifier}/{versions.version_string(info.version)}'
+    file_path = get_malcontent().malcontent_dir_path / item_type / info_path
     return flask.send_file(open(file_path, 'rb'), mimetype='application/json')
 
 @bp.route('/mapping/<string:identifier_dot_json>')
-def get_newest_mapping(identifier_dot_json: str) -> Response:
+def get_newest_mapping(identifier_dot_json: str) -> werkzeug.Response:
     return get_resource_or_mapping('mapping', identifier_dot_json)
 
 @bp.route('/resource/<string:identifier_dot_json>')
-def get_newest_resource(identifier_dot_json: str) -> Response:
+def get_newest_resource(identifier_dot_json: str) -> werkzeug.Response:
     return get_resource_or_mapping('resource', identifier_dot_json)
 
+def make_ref(info: item_infos.AnyInfo) -> dict[str, t.Any]:
+    ref: dict[str, t.Any] = {
+        'version':    info.version,
+        'identifier': info.identifier,
+        'long_name':  info.long_name
+    }
+
+    if isinstance(info, item_infos.ResourceInfo):
+        ref['revision'] = info.revision
+
+    return ref
+
 @bp.route('/query')
 def query():
     url = flask.request.args['url']
 
-    mapping_refs = [i.as_query_result() for i in malcontent().query(url)]
+    mapping_refs = [make_ref(info) for info in get_malcontent().query(url)]
+
     result = {
         '$schema': 'https://hydrilla.koszko.org/schemas/api_query_result-1.schema.json',
         'mappings': mapping_refs,
         'generated_by': generated_by
     }
 
-    return Response(json.dumps(result), mimetype='application/json')
+    return werkzeug.Response(json.dumps(result), mimetype='application/json')
+
+@bp.route('/list_all')
+def list_all_packages():
+    malcontent = get_malcontent()
+
+    resource_refs = [make_ref(info) for info in malcontent.get_all_resources()]
+    mapping_refs  = [make_ref(info) for info in malcontent.get_all_mappings()]
+
+    result = {
+        '$schema': 'https://hydrilla.koszko.org/schemas/api_package_list-2.schema.json',
+        'resources': resource_refs,
+        'mappings':  mapping_refs,
+        'generated_by': generated_by
+    }
+
+    return werkzeug.Response(json.dumps(result), mimetype='application/json')
 
 @bp.route('/--help')
 def mm_help():
@@ -383,8 +219,12 @@ default_project_url = 'https://hydrillabugs.koszko.org/projects/hydrilla/wiki'
 @click.version_option(version=_version.version, prog_name='Hydrilla',
                       message=_('%(prog)s_%(version)s_license'),
                       help=_('version_printing'))
-def start(malcontent_dir: Optional[str], hydrilla_project_url: Optional[str],
-          port: Optional[int], config_path: Optional[str]) -> None:
+def start(
+        malcontent_dir:       t.Optional[str],
+        hydrilla_project_url: t.Optional[str],
+        port:                 t.Optional[int],
+        config_path:          t.Optional[str]
+) -> None:
     """
     Run a development Hydrilla server.