diff options
Diffstat (limited to 'src/hydrilla/server/serve.py')
| -rw-r--r-- | src/hydrilla/server/serve.py | 304 |
1 files changed, 72 insertions, 232 deletions
diff --git a/src/hydrilla/server/serve.py b/src/hydrilla/server/serve.py index 8f0d557..823437a 100644 --- a/src/hydrilla/server/serve.py +++ b/src/hydrilla/server/serve.py @@ -29,221 +29,29 @@ from __future__ import annotations import re import os -import pathlib import json -import logging +import typing as t from pathlib import Path -from hashlib import sha256 -from abc import ABC, abstractmethod -from typing import Optional, Union, Iterable, TypeVar, Generic import click import flask +import werkzeug -from werkzeug import Response - -from .. import _version, versions, json_instances -from ..item_infos import ResourceInfo, MappingInfo, VersionedItemInfo +from ..exceptions import HaketiloException +from .. import _version from ..translations import smart_gettext as _, translation as make_translation -#from ..url_patterns import PatternTree +from .. import versions +from .. import item_infos from . import config +from . import malcontent -here = Path(__file__).resolve().parent generated_by = { 'name': 'hydrilla.server', 'version': _version.version } - # def as_query_result(self) -> dict[str, Union[str, list[int]]]: - # """ - # Produce a json.dump()-able object describing this mapping as one of a - # collection of query results. - # """ - # return { - # 'version': self.version, - # 'identifier': self.identifier, - # 'long_name': self.long_name - # } - -class Malcontent: - """ - Represent a directory with files that can be loaded and served by Hydrilla. - """ - def __init__(self, malcontent_dir_path: Path): - """ - When an instance of Malcontent is constructed, it searches - malcontent_dir_path for serveable site-modifying packages and loads - them into its data structures. - """ - self.resource_infos: dict[str, VersionedItemInfo[ResourceInfo]] = {} - self.mapping_infos: dict[str, VersionedItemInfo[MappingInfo]] = {} - - self.pattern_tree: PatternTree[MappingInfo] = PatternTree() - - self.malcontent_dir_path = malcontent_dir_path - - if not self.malcontent_dir_path.is_dir(): - raise ValueError(_('malcontent_dir_path_not_dir_{}') - .format(malcontent_dir_path)) - - for item_type in ('mapping', 'resource'): - type_path = self.malcontent_dir_path / item_type - if not type_path.is_dir(): - continue - - for subpath in type_path.iterdir(): - if not subpath.is_dir(): - continue - - for ver_file in subpath.iterdir(): - try: - self._load_item(item_type, ver_file) - except Exception as e: - if flask.current_app._hydrilla_werror: - raise e from None - - msg = _('couldnt_load_item_from_{}').format(ver_file) - logging.error(msg, exc_info=True) - - self._report_missing() - self._finalize() - - @staticmethod - def _register_info(infos: dict[str, VersionedItemInfo[VersionedType]], - identifier: str, item_info: VersionedType) -> None: - """ - ........... - """ - infos.setdefault(identifier, VersionedItemInfo())\ - .register(item_info) - - def _load_item(self, item_type: str, ver_file: Path) -> None: - """ - Reads, validates and autocompletes serveable mapping/resource - definition, then registers information from it in data structures. - """ - version = versions.parse_version(ver_file.name) - identifier = ver_file.parent.name - - item_json, major = util.load_instance_from_file(ver_file) - - util.validator_for(f'api_{item_type}_description-{major}.schema.json')\ - .validate(item_json) - - # Assertion needed for mypy. If validation passed, this should not fail. - assert major is not None - - item_info: ItemInfo = ResourceInfo(item_json, major) \ - if item_type == 'resource' else MappingInfo(item_json, major) - - if item_info.identifier != identifier: - msg = _('item_{item}_in_file_{file}')\ - .format({'item': item_info.identifier, 'file': ver_file}) - raise ValueError(msg) - - if item_info.version != version: - ver_str = util.version_string(item_info.version) - msg = _('item_version_{ver}_in_file_{file}')\ - .format({'ver': ver_str, 'file': ver_file}) - raise ValueError(msg) - - if isinstance(item_info, ResourceInfo): - self._register_info(self.resource_infos, identifier, item_info) - elif isinstance(item_info, MappingInfo): - self._register_info(self.mapping_infos, identifier, item_info) - - @staticmethod - def _all_infos(infos: dict[str, VersionedItemInfo[VersionedType]]) \ - -> Iterable[VersionedType]: - """ - ........... - """ - for versioned_info in infos.values(): - for item_info in versioned_info.by_version.values(): - yield item_info - - def _report_missing(self) -> None: - """ - Use logger to print information about items that are referenced but - were not loaded. - """ - def report_missing_dependency(info: ResourceInfo, dep: str) -> None: - msg = _('no_dep_{resource}_{ver}_{dep}')\ - .format(dep=dep, resource=info.identifier, - ver=util.version_string(info.version)) - logging.error(msg) - - for resource_info in self._all_infos(self.resource_infos): - for dep in resource_info.dependencies: - if dep not in self.resource_infos: - report_missing_dependency(resource_info, dep) - - def report_missing_payload(info: MappingInfo, payload: str) -> None: - msg = _('no_payload_{mapping}_{ver}_{payload}')\ - .format(mapping=info.identifier, payload=payload, - ver=util.version_string(info.version)) - logging.error(msg) - - for mapping_info in self._all_infos(self.mapping_infos): - for payload in mapping_info.payloads.values(): - if payload not in self.resource_infos: - report_missing_payload(mapping_info, payload) - - def report_missing_mapping(info: ItemInfo, - required_mapping: str) -> None: - msg = _('no_mapping_{required_by}_{ver}_{required}')\ - .format(required_by=info.identifier, required=required_mapping, - ver=util.version_string(info.version)) - logging.error(msg) - - for item_info in (*self._all_infos(self.mapping_infos), - *self._all_infos(self.resource_infos)): - for required in item_info.required_mappings: - if required not in self.mapping_infos: - report_missing_mapping(item_info, required) - - def _finalize(self): - """ - Initialize structures needed to serve queries. Called once after all - data gets loaded. - """ - for versioned_info in (*self.mapping_infos.values(), - *self.resource_infos.values()): - versioned_info.known_versions.sort() - - for info in self._all_infos(self.mapping_infos): - for pattern in info.payloads: - try: - self.pattern_tree = \ - self.pattern_tree.register(pattern, info) - except Exception as e: - if flask.current_app._hydrilla_werror: - raise e from None - msg = _('couldnt_register_{mapping}_{ver}_{pattern}')\ - .format(mapping=info.identifier, pattern=pattern, - ver=util.version_string(info.version)) - logging.error(msg) - - def query(self, url: str) -> list[MappingInfo]: - """ - Return a list of registered mappings that match url. - - If multiple versions of a mapping are applicable, only the most recent - is included in the result. - """ - collected: dict[str, MappingInfo] = {} - for result_set in self.pattern_tree.search(url): - for wrapped_mapping_info in result_set: - info = wrapped_mapping_info.item - previous = collected.get(info.identifier) - if previous and previous.version > info.version: - continue - - collected[info.identifier] = info - - return list(collected.values()) bp = flask.Blueprint('bp', __package__) @@ -267,41 +75,36 @@ class HydrillaApp(flask.Flask): ] } - self._hydrilla_project_url = hydrilla_config['hydrilla_project_url'] self._hydrilla_port = hydrilla_config['port'] self._hydrilla_werror = hydrilla_config.get('werror', False) + verify_files = hydrilla_config.get('verify_files', True) if 'hydrilla_parent' in hydrilla_config: - raise ValueError("Option 'hydrilla_parent' is not implemented.") + raise HaketiloException(_('err.server.opt_hydrilla_parent_not_implemented')) - malcontent_dir = Path(hydrilla_config['malcontent_dir']).resolve() - with self.app_context(): - self._hydrilla_malcontent = Malcontent(malcontent_dir) + malcontent_dir_path = Path(hydrilla_config['malcontent_dir']).resolve() + self._hydrilla_malcontent = malcontent.Malcontent( + malcontent_dir_path = malcontent_dir_path, + werror = self._hydrilla_werror, + verify_files = verify_files + ) - self.register_blueprint(bp) + self.jinja_env.install_gettext_translations(make_translation()) - def create_jinja_environment(self, *args, **kwargs) \ - -> flask.templating.Environment: - """ - Flask's create_jinja_environment(), but tweaked to always include the - 'hydrilla_project_url' global variable and to install proper - translations. - """ - env = super().create_jinja_environment(*args, **kwargs) # type: ignore - env.install_gettext_translations(make_translation()) - env.globals['hydrilla_project_url'] = self._hydrilla_project_url + self.jinja_env.globals['hydrilla_project_url'] = \ + hydrilla_config['hydrilla_project_url'] - return env + self.register_blueprint(bp) def run(self, *args, **kwargs): """ - Flask's run(), but tweaked to use the port from hydrilla configuration - by default. + Flask's run() but tweaked to use the port from hydrilla configuration by + default. """ return super().run(*args, port=self._hydrilla_port, **kwargs) -def malcontent(): - return flask.current_app._hydrilla_malcontent +def get_malcontent() -> malcontent.Malcontent: + return t.cast(HydrillaApp, flask.current_app)._hydrilla_malcontent @bp.route('/') def index(): @@ -309,7 +112,8 @@ def index(): identifier_json_re = re.compile(r'^([-0-9a-z.]+)\.json$') -def get_resource_or_mapping(item_type: str, identifier: str) -> Response: +def get_resource_or_mapping(item_type: str, identifier: str) \ + -> werkzeug.Response: """ Strip '.json' from 'identifier', look the item up and send its JSON description. @@ -320,41 +124,73 @@ def get_resource_or_mapping(item_type: str, identifier: str) -> Response: identifier = match.group(1) + infos: t.Mapping[str, malcontent.VersionedItemInfo] if item_type == 'resource': - infos = malcontent().resource_infos + infos = get_malcontent().resource_infos else: - infos = malcontent().mapping_infos + infos = get_malcontent().mapping_infos versioned_info = infos.get(identifier) - info = versioned_info and versioned_info.get_by_ver() - if info is None: + if versioned_info is None: flask.abort(404) + info = versioned_info.newest_info() + # no need for send_from_directory(); path is safe, constructed by us - file_path = malcontent().malcontent_dir_path / item_type / info.path() + info_path = f'{info.identifier}/{versions.version_string(info.version)}' + file_path = get_malcontent().malcontent_dir_path / item_type / info_path return flask.send_file(open(file_path, 'rb'), mimetype='application/json') @bp.route('/mapping/<string:identifier_dot_json>') -def get_newest_mapping(identifier_dot_json: str) -> Response: +def get_newest_mapping(identifier_dot_json: str) -> werkzeug.Response: return get_resource_or_mapping('mapping', identifier_dot_json) @bp.route('/resource/<string:identifier_dot_json>') -def get_newest_resource(identifier_dot_json: str) -> Response: +def get_newest_resource(identifier_dot_json: str) -> werkzeug.Response: return get_resource_or_mapping('resource', identifier_dot_json) +def make_ref(info: item_infos.AnyInfo) -> dict[str, t.Any]: + ref: dict[str, t.Any] = { + 'version': info.version, + 'identifier': info.identifier, + 'long_name': info.long_name + } + + if isinstance(info, item_infos.ResourceInfo): + ref['revision'] = info.revision + + return ref + @bp.route('/query') def query(): url = flask.request.args['url'] - mapping_refs = [i.as_query_result() for i in malcontent().query(url)] + mapping_refs = [make_ref(info) for info in get_malcontent().query(url)] + result = { '$schema': 'https://hydrilla.koszko.org/schemas/api_query_result-1.schema.json', 'mappings': mapping_refs, 'generated_by': generated_by } - return Response(json.dumps(result), mimetype='application/json') + return werkzeug.Response(json.dumps(result), mimetype='application/json') + +@bp.route('/list_all') +def list_all_packages(): + malcontent = get_malcontent() + + resource_refs = [make_ref(info) for info in malcontent.get_all_resources()] + mapping_refs = [make_ref(info) for info in malcontent.get_all_mappings()] + + result = { + '$schema': 'https://hydrilla.koszko.org/schemas/api_package_list-2.schema.json', + 'resources': resource_refs, + 'mappings': mapping_refs, + 'generated_by': generated_by + } + + return werkzeug.Response(json.dumps(result), mimetype='application/json') @bp.route('/--help') def mm_help(): @@ -383,8 +219,12 @@ default_project_url = 'https://hydrillabugs.koszko.org/projects/hydrilla/wiki' @click.version_option(version=_version.version, prog_name='Hydrilla', message=_('%(prog)s_%(version)s_license'), help=_('version_printing')) -def start(malcontent_dir: Optional[str], hydrilla_project_url: Optional[str], - port: Optional[int], config_path: Optional[str]) -> None: +def start( + malcontent_dir: t.Optional[str], + hydrilla_project_url: t.Optional[str], + port: t.Optional[int], + config_path: t.Optional[str] +) -> None: """ Run a development Hydrilla server. |