From d54a95e0f9c689f2bbaaea90a3a16a855a408823 Mon Sep 17 00:00:00 2001 From: Wojtek Kosior Date: Wed, 17 Aug 2022 13:50:34 +0200 Subject: allow loading packages from zip files through web UI and listing installed mappings --- src/hydrilla/proxy/policies/payload.py | 24 +- src/hydrilla/proxy/policies/payload_resource.py | 11 +- src/hydrilla/proxy/state.py | 48 ++- src/hydrilla/proxy/state_impl/concrete_state.py | 402 ++------------------- src/hydrilla/proxy/state_impl/load_packages.py | 344 ++++++++++++++++++ src/hydrilla/proxy/state_impl/mappings.py | 133 +++++++ src/hydrilla/proxy/tables.sql | 55 ++- src/hydrilla/proxy/web_ui/_app.py | 13 + src/hydrilla/proxy/web_ui/packages.py | 89 ++++- src/hydrilla/proxy/web_ui/root.py | 19 +- .../proxy/web_ui/templates/base.html.jinja | 14 +- .../proxy/web_ui/templates/packages.html.jinja | 78 ++++ .../templates/packages__load_from_disk.html.jinja | 14 +- .../templates/packages__show_single.html.jinja | 32 ++ 14 files changed, 859 insertions(+), 417 deletions(-) create mode 100644 src/hydrilla/proxy/state_impl/load_packages.py create mode 100644 src/hydrilla/proxy/state_impl/mappings.py create mode 100644 src/hydrilla/proxy/web_ui/_app.py create mode 100644 src/hydrilla/proxy/web_ui/templates/packages.html.jinja create mode 100644 src/hydrilla/proxy/web_ui/templates/packages__show_single.html.jinja (limited to 'src/hydrilla/proxy') diff --git a/src/hydrilla/proxy/policies/payload.py b/src/hydrilla/proxy/policies/payload.py index 577c94e..e64653d 100644 --- a/src/hydrilla/proxy/policies/payload.py +++ b/src/hydrilla/proxy/policies/payload.py @@ -46,7 +46,6 @@ from . import base @dc.dataclass(frozen=True) # type: ignore[misc] class PayloadAwarePolicy(base.Policy): """....""" - haketilo_state: state.HaketiloState payload_data: state.PayloadData def assets_base_url(self, request_url: ParsedUrl): @@ -174,7 +173,7 @@ class PayloadInjectPolicy(PayloadAwarePolicy): base_url = self.assets_base_url(url) payload_ref = self.payload_data.payload_ref - for path in payload_ref.get_script_paths(self.haketilo_state): + for path in payload_ref.get_script_paths(): yield base_url + '/'.join(('static', *path)) def _modify_body( @@ -266,19 +265,6 @@ class AutoPayloadInjectPolicy(PayloadInjectPolicy): """....""" priority: t.ClassVar[base.PolicyPriority] = base.PolicyPriority._ONE - def _modify_body( - self, - url: ParsedUrl, - body: bytes, - encoding: t.Optional[str] - ) -> bytes: - """....""" - payload_ref = self.payload_data.payload_ref - mapping_ref = payload_ref.get_mapping(self.haketilo_state) - mapping_ref.enable(self.haketilo_state) - - return super()._modify_body(url, body, encoding) - @dc.dataclass(frozen=True) class PayloadSuggestPolicy(PayloadAwarePolicy): @@ -299,20 +285,20 @@ class PayloadPolicyFactory(PayloadAwarePolicyFactory): -> t.Optional[base.Policy]: """....""" try: - payload_data = self.payload_ref.get_data(haketilo_state) + payload_data = self.payload_ref.get_data() except: return None if payload_data.explicitly_enabled: - return PayloadInjectPolicy(haketilo_state, payload_data) + return PayloadInjectPolicy(payload_data) mode = haketilo_state.get_settings().mapping_use_mode if mode == state.MappingUseMode.QUESTION: - return PayloadSuggestPolicy(haketilo_state, payload_data) + return PayloadSuggestPolicy(payload_data) if mode == state.MappingUseMode.WHEN_ENABLED: return None # mode == state.MappingUseMode.AUTO - return AutoPayloadInjectPolicy(haketilo_state, payload_data) + return AutoPayloadInjectPolicy(payload_data) diff --git a/src/hydrilla/proxy/policies/payload_resource.py b/src/hydrilla/proxy/policies/payload_resource.py index 3e1b31a..3aacea0 100644 --- a/src/hydrilla/proxy/policies/payload_resource.py +++ b/src/hydrilla/proxy/policies/payload_resource.py @@ -77,10 +77,7 @@ class PayloadResourcePolicy(PayloadAwarePolicy): -> http_messages.ProducedResponse: """....""" try: - file_data = self.payload_data.payload_ref.get_file_data( - self.haketilo_state, - path - ) + file_data = self.payload_data.payload_ref.get_file_data(path) except state.MissingItemError: return resource_blocked_response @@ -147,7 +144,7 @@ class PayloadResourcePolicyFactory(PayloadAwarePolicyFactory): -> t.Union[PayloadResourcePolicy, BlockedResponsePolicy]: """....""" try: - payload_data = self.payload_ref.get_data(haketilo_state) + payload_data = self.payload_ref.get_data() except state.MissingItemError: return BlockedResponsePolicy() @@ -156,6 +153,4 @@ class PayloadResourcePolicyFactory(PayloadAwarePolicyFactory): state.MappingUseMode.AUTO: return BlockedResponsePolicy() - return PayloadResourcePolicy(haketilo_state, payload_data) - - + return PayloadResourcePolicy(payload_data) diff --git a/src/hydrilla/proxy/state.py b/src/hydrilla/proxy/state.py index f511056..14d38b6 100644 --- a/src/hydrilla/proxy/state.py +++ b/src/hydrilla/proxy/state.py @@ -44,6 +44,7 @@ from immutables import Map from ..versions import VerTuple from ..url_patterns import ParsedPattern +from .. import item_infos class EnabledStatus(Enum): @@ -68,6 +69,14 @@ class Ref: id: str +RefType = t.TypeVar('RefType', bound=Ref) + +class Store(ABC, t.Generic[RefType]): + @abstractmethod + def get(self, id) -> RefType: + ... + + # mypy needs to be corrected: # https://stackoverflow.com/questions/70999513/conflict-between-mix-ins-for-abstract-dataclasses/70999704#70999704 @dc.dataclass(frozen=True, unsafe_hash=True) # type: ignore[misc] @@ -114,15 +123,35 @@ class MappingRef(Ref): """....""" ... +class MappingStore(Store[MappingRef]): + pass + @dc.dataclass(frozen=True, unsafe_hash=True) # type: ignore[misc] class MappingVersionRef(Ref): """....""" @abstractmethod - def enable(self, state: 'HaketiloState') -> None: + def update_status(self, new_status: EnabledStatus) -> None: """....""" ... + @abstractmethod + def get_display_info(self) -> MappingDisplayInfo: + ... + +@dc.dataclass(frozen=True) +class MappingDisplayInfo: + ref: MappingVersionRef + info: item_infos.MappingInfo + enabled: EnabledStatus + is_orphan: bool + +class MappingVersionStore(Store[MappingVersionRef]): + @abstractmethod + def get_display_infos(self) -> t.Iterable[MappingDisplayInfo]: + ... + + @dc.dataclass(frozen=True, unsafe_hash=True) # type: ignore[misc] class ResourceRef(Ref): """....""" @@ -167,23 +196,23 @@ class FileData: class PayloadRef(Ref): """....""" @abstractmethod - def get_data(self, state: 'HaketiloState') -> PayloadData: + def get_data(self) -> PayloadData: """....""" ... @abstractmethod - def get_mapping(self, state: 'HaketiloState') -> MappingVersionRef: + def get_mapping(self) -> MappingVersionRef: """....""" ... @abstractmethod - def get_script_paths(self, state: 'HaketiloState') \ + def get_script_paths(self) \ -> t.Iterable[t.Sequence[str]]: """....""" ... @abstractmethod - def get_file_data(self, state: 'HaketiloState', path: t.Sequence[str]) \ + def get_file_data(self, path: t.Sequence[str]) \ -> t.Optional[FileData]: """....""" ... @@ -219,6 +248,10 @@ class MissingItemError(ValueError): class HaketiloState(ABC): """....""" + @abstractmethod + def import_packages(self, malcontent_path: Path) -> None: + ... + @abstractmethod def get_repo(self, repo_id: str) -> RepoRef: """....""" @@ -230,13 +263,12 @@ class HaketiloState(ABC): ... @abstractmethod - def get_mapping(self, mapping_id: str) -> MappingRef: + def mapping_store(self) -> MappingStore: """....""" ... @abstractmethod - def get_mapping_version(self, mapping_version_id: str) \ - -> MappingVersionRef: + def mapping_version_store(self) -> MappingVersionStore: """....""" ... diff --git a/src/hydrilla/proxy/state_impl/concrete_state.py b/src/hydrilla/proxy/state_impl/concrete_state.py index bb14734..b2b1033 100644 --- a/src/hydrilla/proxy/state_impl/concrete_state.py +++ b/src/hydrilla/proxy/state_impl/concrete_state.py @@ -46,12 +46,13 @@ from ...exceptions import HaketiloException from ...translations import smart_gettext as _ from ... import pattern_tree from ... import url_patterns -from ... import versions from ... import item_infos from ..simple_dependency_satisfying import compute_payloads, ComputedPayload from .. import state as st from .. import policies from . import base +from . import mappings +from .load_packages import load_packages here = Path(__file__).resolve().parent @@ -79,21 +80,6 @@ class ConcreteRepoIterationRef(st.RepoIterationRef): pass -@dc.dataclass(frozen=True, unsafe_hash=True) -class ConcreteMappingRef(st.MappingRef): - def disable(self, state: st.HaketiloState) -> None: - raise NotImplementedError() - - def forget_enabled(self, state: st.HaketiloState) -> None: - raise NotImplementedError() - - -@dc.dataclass(frozen=True, unsafe_hash=True) -class ConcreteMappingVersionRef(st.MappingVersionRef): - def enable(self, state: st.HaketiloState) -> None: - raise NotImplementedError() - - @dc.dataclass(frozen=True, unsafe_hash=True) class ConcreteResourceRef(st.ResourceRef): pass @@ -106,15 +92,20 @@ class ConcreteResourceVersionRef(st.ResourceVersionRef): @dc.dataclass(frozen=True, unsafe_hash=True) class ConcretePayloadRef(st.PayloadRef): - def get_data(self, state: st.HaketiloState) -> st.PayloadData: - return t.cast(ConcreteHaketiloState, state).payloads_data[self] + state: base.HaketiloStateWithFields = dc.field(hash=False, compare=False) - def get_mapping(self, state: st.HaketiloState) -> st.MappingVersionRef: - return 'to implement' + def get_data(self) -> st.PayloadData: + try: + return self.state.payloads_data[self] + except KeyError: + raise st.MissingItemError() + + def get_mapping(self) -> st.MappingVersionRef: + raise NotImplementedError() - def get_script_paths(self, state: st.HaketiloState) \ + def get_script_paths(self) \ -> t.Iterable[t.Sequence[str]]: - with t.cast(ConcreteHaketiloState, state).cursor() as cursor: + with self.state.cursor() as cursor: cursor.execute( ''' SELECT @@ -153,7 +144,7 @@ class ConcretePayloadRef(st.PayloadRef): return paths - def get_file_data(self, state: st.HaketiloState, path: t.Sequence[str]) \ + def get_file_data(self, path: t.Sequence[str]) \ -> t.Optional[st.FileData]: if len(path) == 0: raise st.MissingItemError() @@ -162,7 +153,7 @@ class ConcretePayloadRef(st.PayloadRef): file_name = '/'.join(file_name_segments) - with t.cast(ConcreteHaketiloState, state).cursor() as cursor: + with self.state.cursor() as cursor: cursor.execute( ''' SELECT @@ -197,61 +188,6 @@ class ConcretePayloadRef(st.PayloadRef): return st.FileData(type=mime_type, name=file_name, contents=data) -# @dc.dataclass(frozen=True, unsafe_hash=True) -# class ConcretePayloadRef(st.PayloadRef): -# computed_payload: ComputedPayload = dc.field(hash=False, compare=False) - -# def get_data(self, state: st.HaketiloState) -> st.PayloadData: -# return t.cast(ConcreteHaketiloState, state).payloads_data[self.id] - -# def get_mapping(self, state: st.HaketiloState) -> st.MappingVersionRef: -# return 'to implement' - -# def get_script_paths(self, state: st.HaketiloState) \ -# -> t.Iterator[t.Sequence[str]]: -# for resource_info in self.computed_payload.resources: -# for file_spec in resource_info.scripts: -# yield (resource_info.identifier, *file_spec.name.split('/')) - -# def get_file_data(self, state: st.HaketiloState, path: t.Sequence[str]) \ -# -> t.Optional[st.FileData]: -# if len(path) == 0: -# raise st.MissingItemError() - -# resource_identifier, *file_name_segments = path - -# file_name = '/'.join(file_name_segments) - -# script_sha256 = '' - -# matched_resource_info = False - -# for resource_info in self.computed_payload.resources: -# if resource_info.identifier == resource_identifier: -# matched_resource_info = True - -# for script_spec in resource_info.scripts: -# if script_spec.name == file_name: -# script_sha256 = script_spec.sha256 - -# break - -# if not matched_resource_info: -# raise st.MissingItemError(resource_identifier) - -# if script_sha256 == '': -# return None - -# store_dir_path = t.cast(ConcreteHaketiloState, state).store_dir -# files_dir_path = store_dir_path / 'temporary_malcontent' / 'file' -# file_path = files_dir_path / 'sha256' / script_sha256 - -# return st.FileData( -# type = 'application/javascript', -# name = file_name, -# contents = file_path.read_bytes() -# ) - def register_payload( policy_tree: base.PolicyTree, pattern: url_patterns.ParsedPattern, @@ -278,205 +214,12 @@ def register_payload( return policy_tree -DataById = t.Mapping[str, st.PayloadData] - AnyInfoVar = t.TypeVar( 'AnyInfoVar', item_infos.ResourceInfo, item_infos.MappingInfo ) -def read_items(malcontent_path: Path, item_class: t.Type[AnyInfoVar]) \ - -> t.Iterator[tuple[AnyInfoVar, str]]: - item_type_path = malcontent_path / item_class.type_name - if not item_type_path.is_dir(): - return - - for item_path in item_type_path.iterdir(): - if not item_path.is_dir(): - continue - - for item_version_path in item_path.iterdir(): - definition = item_version_path.read_text() - item_info = item_class.load(io.StringIO(definition)) - - assert item_info.identifier == item_path.name - assert versions.version_string(item_info.version) == \ - item_version_path.name - - yield item_info, definition - -def get_or_make_repo_iteration(cursor: sqlite3.Cursor, repo_name: str) -> int: - cursor.execute( - ''' - INSERT OR IGNORE INTO repos(name, url, deleted, next_iteration) - VALUES(?, '', TRUE, 2); - ''', - (repo_name,) - ) - - cursor.execute( - ''' - SELECT - repo_id, next_iteration - 1 - FROM - repos - WHERE - name = ?; - ''', - (repo_name,) - ) - - (repo_id, last_iteration), = cursor.fetchall() - - cursor.execute( - ''' - INSERT OR IGNORE INTO repo_iterations(repo_id, iteration) - VALUES(?, ?); - ''', - (repo_id, last_iteration) - ) - - cursor.execute( - ''' - SELECT - repo_iteration_id - FROM - repo_iterations - WHERE - repo_id = ? AND iteration = ?; - ''', - (repo_id, last_iteration) - ) - - (repo_iteration_id,), = cursor.fetchall() - - return repo_iteration_id - -def get_or_make_item(cursor: sqlite3.Cursor, type: str, identifier: str) -> int: - type_letter = {'resource': 'R', 'mapping': 'M'}[type] - - cursor.execute( - ''' - INSERT OR IGNORE INTO items(type, identifier) - VALUES(?, ?); - ''', - (type_letter, identifier) - ) - - cursor.execute( - ''' - SELECT - item_id - FROM - items - WHERE - type = ? AND identifier = ?; - ''', - (type_letter, identifier) - ) - - (item_id,), = cursor.fetchall() - - return item_id - -def get_or_make_item_version( - cursor: sqlite3.Cursor, - item_id: int, - repo_iteration_id: int, - version: versions.VerTuple, - definition: str -) -> int: - ver_str = versions.version_string(version) - - cursor.execute( - ''' - INSERT OR IGNORE INTO item_versions( - item_id, - version, - repo_iteration_id, - definition - ) - VALUES(?, ?, ?, ?); - ''', - (item_id, ver_str, repo_iteration_id, definition) - ) - - cursor.execute( - ''' - SELECT - item_version_id - FROM - item_versions - WHERE - item_id = ? AND version = ? AND repo_iteration_id = ?; - ''', - (item_id, ver_str, repo_iteration_id) - ) - - (item_version_id,), = cursor.fetchall() - - return item_version_id - -def make_mapping_status(cursor: sqlite3.Cursor, item_id: int) -> None: - cursor.execute( - ''' - INSERT OR IGNORE INTO mapping_statuses(item_id, enabled) - VALUES(?, 'N'); - ''', - (item_id,) - ) - -def get_or_make_file(cursor: sqlite3.Cursor, sha256: str, file_bytes: bytes) \ - -> int: - cursor.execute( - ''' - INSERT OR IGNORE INTO files(sha256, data) - VALUES(?, ?) - ''', - (sha256, file_bytes) - ) - - cursor.execute( - ''' - SELECT - file_id - FROM - files - WHERE - sha256 = ?; - ''', - (sha256,) - ) - - (file_id,), = cursor.fetchall() - - return file_id - -def make_file_use( - cursor: sqlite3.Cursor, - item_version_id: int, - file_id: int, - name: str, - type: str, - mime_type: str, - idx: int -) -> None: - cursor.execute( - ''' - INSERT OR IGNORE INTO file_uses( - item_version_id, - file_id, - name, - type, - mime_type, - idx - ) - VALUES(?, ?, ?, ?, ?, ?); - ''', - (item_version_id, file_id, name, type, mime_type, idx) - ) - def get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar],) \ -> t.Mapping[AnyInfoVar, int]: cursor.execute( @@ -508,10 +251,7 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields): def __post_init__(self) -> None: self._prepare_database() - self._populate_database_with_stuff_from_temporary_malcontent_dir() - - with self.cursor(transaction=True) as cursor: - self.recompute_payloads(cursor) + self._rebuild_structures() def _prepare_database(self) -> None: """....""" @@ -546,7 +286,7 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields): (db_haketilo_version,) = cursor.fetchone() if db_haketilo_version != '3.0b1': - raise HaketiloException(_('err.unknown_db_schema')) + raise HaketiloException(_('err.proxy.unknown_db_schema')) cursor.execute('PRAGMA FOREIGN_KEYS;') if cursor.fetchall() == []: @@ -556,88 +296,10 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields): finally: cursor.close() - def _populate_database_with_stuff_from_temporary_malcontent_dir(self) \ - -> None: - malcontent_dir_path = self.store_dir / 'temporary_malcontent' - files_by_sha256_path = malcontent_dir_path / 'file' / 'sha256' - + def import_packages(self, malcontent_path: Path) -> None: with self.cursor(transaction=True) as cursor: - for info_type in [item_infos.ResourceInfo, item_infos.MappingInfo]: - info: item_infos.AnyInfo - for info, definition in read_items( - malcontent_dir_path, - info_type # type: ignore - ): - repo_iteration_id = get_or_make_repo_iteration( - cursor, - info.repo - ) - - item_id = get_or_make_item( - cursor, - info.type_name, - info.identifier - ) - - item_version_id = get_or_make_item_version( - cursor, - item_id, - repo_iteration_id, - info.version, - definition - ) - - if info_type is item_infos.MappingInfo: - make_mapping_status(cursor, item_id) - - file_ids_bytes = {} - - file_specifiers = [*info.source_copyright] - if isinstance(info, item_infos.ResourceInfo): - file_specifiers.extend(info.scripts) - - for file_spec in file_specifiers: - file_path = files_by_sha256_path / file_spec.sha256 - file_bytes = file_path.read_bytes() - - sha256 = hashlib.sha256(file_bytes).digest().hex() - assert sha256 == file_spec.sha256 - - file_id = get_or_make_file(cursor, sha256, file_bytes) - - file_ids_bytes[sha256] = (file_id, file_bytes) - - for idx, file_spec in enumerate(info.source_copyright): - file_id, file_bytes = file_ids_bytes[file_spec.sha256] - if file_bytes.isascii(): - mime = 'text/plain' - else: - mime = 'application/octet-stream' - - make_file_use( - cursor, - item_version_id = item_version_id, - file_id = file_id, - name = file_spec.name, - type = 'L', - mime_type = mime, - idx = idx - ) - - if isinstance(info, item_infos.MappingInfo): - continue - - for idx, file_spec in enumerate(info.scripts): - file_id, _ = file_ids_bytes[file_spec.sha256] - make_file_use( - cursor, - item_version_id = item_version_id, - file_id = file_id, - name = file_spec.name, - type = 'W', - mime_type = 'application/javascript', - idx = idx - ) + load_packages(self, cursor, malcontent_path) + self.recompute_payloads(cursor) def recompute_payloads(self, cursor: sqlite3.Cursor) -> None: assert self.connection.in_transaction @@ -700,7 +362,16 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields): self._rebuild_structures(cursor) - def _rebuild_structures(self, cursor: sqlite3.Cursor) -> None: + def _rebuild_structures(self, cursor: t.Optional[sqlite3.Cursor] = None) \ + -> None: + """ + Recreation of data structures as done after every recomputation of + dependencies as well as at startup. + """ + if cursor is None: + with self.cursor() as new_cursor: + return self._rebuild_structures(new_cursor) + cursor.execute( ''' SELECT @@ -734,7 +405,7 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields): enabled_status, identifier) = row - payload_ref = ConcretePayloadRef(str(payload_id_int)) + payload_ref = ConcretePayloadRef(str(payload_id_int), self) previous_data = self.payloads_data.get(payload_ref) if previous_data is not None: @@ -775,12 +446,11 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields): def get_repo_iteration(self, repo_iteration_id: str) -> st.RepoIterationRef: return ConcreteRepoIterationRef(repo_iteration_id) - def get_mapping(self, mapping_id: str) -> st.MappingRef: - return ConcreteMappingRef(mapping_id) + def mapping_store(self) -> st.MappingStore: + raise NotImplementedError() - def get_mapping_version(self, mapping_version_id: str) \ - -> st.MappingVersionRef: - return ConcreteMappingVersionRef(mapping_version_id) + def mapping_version_store(self) -> st.MappingVersionStore: + return mappings.ConcreteMappingVersionStore(self) def get_resource(self, resource_id: str) -> st.ResourceRef: return ConcreteResourceRef(resource_id) @@ -790,7 +460,7 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields): return ConcreteResourceVersionRef(resource_version_id) def get_payload(self, payload_id: str) -> st.PayloadRef: - return 'not implemented' + raise NotImplementedError() def add_repo(self, name: t.Optional[str], url: t.Optional[str]) \ -> st.RepoRef: diff --git a/src/hydrilla/proxy/state_impl/load_packages.py b/src/hydrilla/proxy/state_impl/load_packages.py new file mode 100644 index 0000000..6983c3e --- /dev/null +++ b/src/hydrilla/proxy/state_impl/load_packages.py @@ -0,0 +1,344 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +# Haketilo proxy data and configuration (import of packages from disk files). +# +# This file is part of Hydrilla&Haketilo. +# +# Copyright (C) 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this +# file's license. Although I request that you do not make use this code +# in a proprietary program, I am not going to enforce this in court. + +""" +.... +""" + +# Enable using with Python 3.7. +from __future__ import annotations + +import io +import hashlib +import dataclasses as dc +import typing as t + +from pathlib import Path + +import sqlite3 + +from ...exceptions import HaketiloException +from ...translations import smart_gettext as _ +from ... import versions +from ... import item_infos +from . import base + + +def get_or_make_repo_iteration(cursor: sqlite3.Cursor, repo_name: str) -> int: + cursor.execute( + ''' + SELECT + repo_id, next_iteration - 1 + FROM + repos + WHERE + name = ?; + ''', + (repo_name,) + ) + + (repo_id, last_iteration), = cursor.fetchall() + + cursor.execute( + ''' + INSERT OR IGNORE INTO repo_iterations(repo_id, iteration) + VALUES(?, ?); + ''', + (repo_id, last_iteration) + ) + + cursor.execute( + ''' + SELECT + repo_iteration_id + FROM + repo_iterations + WHERE + repo_id = ? AND iteration = ?; + ''', + (repo_id, last_iteration) + ) + + (repo_iteration_id,), = cursor.fetchall() + + return repo_iteration_id + +def get_or_make_item(cursor: sqlite3.Cursor, type: str, identifier: str) -> int: + type_letter = {'resource': 'R', 'mapping': 'M'}[type] + + cursor.execute( + ''' + INSERT OR IGNORE INTO items(type, identifier) + VALUES(?, ?); + ''', + (type_letter, identifier) + ) + + cursor.execute( + ''' + SELECT + item_id + FROM + items + WHERE + type = ? AND identifier = ?; + ''', + (type_letter, identifier) + ) + + (item_id,), = cursor.fetchall() + + return item_id + +def get_or_make_item_version( + cursor: sqlite3.Cursor, + item_id: int, + repo_iteration_id: int, + version: versions.VerTuple, + definition: str +) -> int: + ver_str = versions.version_string(version) + + cursor.execute( + ''' + INSERT OR IGNORE INTO item_versions( + item_id, + version, + repo_iteration_id, + definition + ) + VALUES(?, ?, ?, ?); + ''', + (item_id, ver_str, repo_iteration_id, definition) + ) + + cursor.execute( + ''' + SELECT + item_version_id + FROM + item_versions + WHERE + item_id = ? AND version = ? AND repo_iteration_id = ?; + ''', + (item_id, ver_str, repo_iteration_id) + ) + + (item_version_id,), = cursor.fetchall() + + return item_version_id + +def make_mapping_status(cursor: sqlite3.Cursor, item_id: int) -> None: + cursor.execute( + ''' + INSERT OR IGNORE INTO mapping_statuses(item_id, enabled, frozen) + VALUES(?, 'E', 'R'); + ''', + (item_id,) + ) + +def get_or_make_file(cursor: sqlite3.Cursor, sha256: str, file_bytes: bytes) \ + -> int: + cursor.execute( + ''' + INSERT OR IGNORE INTO files(sha256, data) + VALUES(?, ?) + ''', + (sha256, file_bytes) + ) + + cursor.execute( + ''' + SELECT + file_id + FROM + files + WHERE + sha256 = ?; + ''', + (sha256,) + ) + + (file_id,), = cursor.fetchall() + + return file_id + +def make_file_use( + cursor: sqlite3.Cursor, + item_version_id: int, + file_id: int, + name: str, + type: str, + mime_type: str, + idx: int +) -> None: + cursor.execute( + ''' + INSERT OR IGNORE INTO file_uses( + item_version_id, + file_id, + name, + type, + mime_type, + idx + ) + VALUES(?, ?, ?, ?, ?, ?); + ''', + (item_version_id, file_id, name, type, mime_type, idx) + ) + +@dc.dataclass(frozen=True) +class _FileInfo: + id: int + is_ascii: bool + +def _add_item( + cursor: sqlite3.Cursor, + files_by_sha256_path: Path, + info: item_infos.AnyInfo, + definition: str +) -> None: + repo_iteration_id = get_or_make_repo_iteration(cursor, '') + + item_id = get_or_make_item(cursor, info.type_name, info.identifier) + + item_version_id = get_or_make_item_version( + cursor, + item_id, + repo_iteration_id, + info.version, + definition + ) + + if isinstance(info, item_infos.MappingInfo): + make_mapping_status(cursor, item_id) + + file_infos = {} + + file_specifiers = [*info.source_copyright] + if isinstance(info, item_infos.ResourceInfo): + file_specifiers.extend(info.scripts) + + for file_spec in file_specifiers: + file_path = files_by_sha256_path / file_spec.sha256 + if not file_path.is_file(): + fmt = _('err.proxy.file_missing_{item_identifier}_{file_name}_{sha256}') + msg = fmt.format( + item_identifier = info.identifier, + file_name = file_spec.name, + sha256 = file_spec.sha256 + ) + raise HaketiloException(msg) + + file_bytes = file_path.read_bytes() + + sha256 = hashlib.sha256(file_bytes).digest().hex() + if sha256 != file_spec.sha256: + fmt = _('err.proxy.file_hash_mismatched_{item_identifier}_{file_name}_{expected_sha256}_{actual_sha256}') + msg = fmt.format( + item_identifier = info.identifier, + file_name = file_spec.name, + expected_sha256 = file_spec.sha256, + actual_sha256 = sha256 + ) + raise HaketiloException(msg) + + file_id = get_or_make_file(cursor, sha256, file_bytes) + + file_infos[sha256] = _FileInfo(file_id, file_bytes.isascii()) + + for idx, file_spec in enumerate(info.source_copyright): + file_info = file_infos[file_spec.sha256] + if file_info.is_ascii: + mime = 'text/plain' + else: + mime = 'application/octet-stream' + + make_file_use( + cursor, + item_version_id = item_version_id, + file_id = file_info.id, + name = file_spec.name, + type = 'L', + mime_type = mime, + idx = idx + ) + + if isinstance(info, item_infos.MappingInfo): + return + + for idx, file_spec in enumerate(info.scripts): + file_info = file_infos[file_spec.sha256] + make_file_use( + cursor, + item_version_id = item_version_id, + file_id = file_info.id, + name = file_spec.name, + type = 'W', + mime_type = 'application/javascript', + idx = idx + ) + +AnyInfoVar = t.TypeVar( + 'AnyInfoVar', + item_infos.ResourceInfo, + item_infos.MappingInfo +) + +def _read_items(malcontent_path: Path, item_class: t.Type[AnyInfoVar]) \ + -> t.Iterator[tuple[AnyInfoVar, str]]: + item_type_path = malcontent_path / item_class.type_name + if not item_type_path.is_dir(): + return + + for item_path in item_type_path.iterdir(): + if not item_path.is_dir(): + continue + + for item_version_path in item_path.iterdir(): + definition = item_version_path.read_text() + item_info = item_class.load(io.StringIO(definition)) + + assert item_info.identifier == item_path.name + assert versions.version_string(item_info.version) == \ + item_version_path.name + + yield item_info, definition + +def load_packages( + state: base.HaketiloStateWithFields, + cursor: sqlite3.Cursor, + malcontent_path: Path +) -> None: + files_by_sha256_path = malcontent_path / 'file' / 'sha256' + + for info_type in [item_infos.ResourceInfo, item_infos.MappingInfo]: + info: item_infos.AnyInfo + for info, definition in _read_items( + malcontent_path, + info_type # type: ignore + ): + _add_item(cursor, files_by_sha256_path, info, definition) diff --git a/src/hydrilla/proxy/state_impl/mappings.py b/src/hydrilla/proxy/state_impl/mappings.py new file mode 100644 index 0000000..5e31814 --- /dev/null +++ b/src/hydrilla/proxy/state_impl/mappings.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +# Haketilo proxy data and configuration (MappingRef and MappingStore subtypes). +# +# This file is part of Hydrilla&Haketilo. +# +# Copyright (C) 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this +# file's license. Although I request that you do not make use this code +# in a proprietary program, I am not going to enforce this in court. + +""" +This module provides an interface to interact with mappings inside Haketilo. +""" + +# Enable using with Python 3.7. +from __future__ import annotations + +import io +import typing as t +import dataclasses as dc + +from ... import item_infos +from .. import state as st +from . import base + + +@dc.dataclass(frozen=True, unsafe_hash=True) +class ConcreteMappingVersionRef(st.MappingVersionRef): + """....""" + state: base.HaketiloStateWithFields = dc.field(hash=False, compare=False) + + def update_status(self, new_status: st.EnabledStatus) -> None: + """....""" + assert new_status != st.EnabledStatus.AUTO_ENABLED + raise NotImplementedError() + + def get_display_info(self) -> st.MappingDisplayInfo: + with self.state.cursor() as cursor: + cursor.execute( + ''' + SELECT + enabled, + definition, + repo, + repo_iteration, + is_orphan + FROM + mapping_display_infos + WHERE + item_version_id = ?; + ''', + (self.id,) + ) + + rows = cursor.fetchall() + + if rows == []: + raise st.MissingItemError() + + (status_letter, definition, repo, repo_iteration, is_orphan), = rows + + item_info = item_infos.MappingInfo.load( + io.StringIO(definition), + repo, + repo_iteration + ) + + status = st.EnabledStatus(status_letter) + + return st.MappingDisplayInfo(self, item_info, status, is_orphan) + + +@dc.dataclass(frozen=True) +class ConcreteMappingVersionStore(st.MappingVersionStore): + state: base.HaketiloStateWithFields + + def get(self, id: str) -> st.MappingVersionRef: + return ConcreteMappingVersionRef(id, self.state) + + def get_display_infos(self) -> t.Iterable[st.MappingDisplayInfo]: + with self.state.cursor() as cursor: + cursor.execute( + ''' + SELECT + enabled, + item_version_id, + definition, + repo, + repo_iteration, + is_orphan + FROM + mapping_display_infos; + ''' + ) + + all_rows = cursor.fetchall() + + result = [] + + for row in all_rows: + (status_letter, item_version_id, definition, repo, repo_iteration, + is_orphan) = row + + ref = ConcreteMappingVersionRef(str(item_version_id), self.state) + + item_info = item_infos.MappingInfo.load( + io.StringIO(definition), + repo, + repo_iteration + ) + + status = st.EnabledStatus(status_letter) + + info = st.MappingDisplayInfo(ref, item_info, status, is_orphan) + result.append(info) + + return result diff --git a/src/hydrilla/proxy/tables.sql b/src/hydrilla/proxy/tables.sql index 2a6cac6..0417613 100644 --- a/src/hydrilla/proxy/tables.sql +++ b/src/hydrilla/proxy/tables.sql @@ -45,7 +45,8 @@ INSERT INTO general( default_allow_scripts, repo_refresh_seconds, mapping_use_mode -) VALUES( +) +VALUES( 1, '3.0b1', FALSE, @@ -66,19 +67,26 @@ CREATE TABLE repos( repo_id INTEGER PRIMARY KEY, name VARCHAR NOT NULL, - url VARCHAR NOT NULL, - deleted BOOLEAN NOT NULL, + url VARCHAR NULL, + deleted BOOLEAN NULL, next_iteration INTEGER NOT NULL, active_iteration_id INTEGER NULL, last_refreshed INTEGER NULL, UNIQUE (name), + CHECK ((repo_id = 1) = (name = '')), + CHECK ((repo_id = 1) = (url IS NULL)), + CHECK ((repo_id = 1) = (deleted IS NULL)), + CHECK (repo_id != 1 OR last_refreshed IS NULL), FOREIGN KEY (active_iteration_id) REFERENCES repo_iterations(repo_iteration_id) ON DELETE SET NULL ); +INSERT INTO repos(repo_id, name, next_iteration) +VALUES(1, '', 1); + CREATE TABLE repo_iterations( repo_iteration_id INTEGER PRIMARY KEY, @@ -126,8 +134,9 @@ CREATE TABLE mapping_statuses( -- REPOSITORY or is NOT_FROZEN at all. frozen CHAR(1) NULL, - CHECK ((frozen IS NULL) = (enabled != 'E')), - CHECK (frozen IS NULL OR frozen in ('E', 'R', 'N')) + CHECK (enabled IN ('E', 'D', 'N')), + CHECK ((frozen IS NULL) = (enabled != 'E')), + CHECK (frozen IS NULL OR frozen in ('E', 'R', 'N')) ); CREATE TABLE item_versions( @@ -163,6 +172,42 @@ CREATE TABLE payloads( ON DELETE CASCADE ); +CREATE VIEW mapping_display_infos +AS +SELECT + CASE WHEN + ms.enabled = 'N' AND COUNT(p.payload_id) > 0 + THEN + 'A' -- AUTO_ENABLED mapping + ELSE + ms.enabled + END AS enabled, + iv.item_version_id, iv.definition, + r.name AS repo, + ri.iteration AS repo_iteration, + COALESCE( + r.active_iteration_id != ri.repo_iteration_id, + TRUE + ) AND r.name != '' AS is_orphan +FROM + item_versions AS iv + LEFT JOIN payloads AS p + ON iv.item_version_id = p.mapping_item_id + JOIN items AS i + USING (item_id) + JOIN mapping_statuses AS ms + USING (item_id) + JOIN repo_iterations AS ri + USING (repo_iteration_id) + JOIN repos AS r + USING (repo_id) +WHERE + i.type = 'M' +GROUP BY + ms.enabled, + iv.item_version_id, iv.definition, + r.name, ri.iteration; + CREATE TABLE resolved_depended_resources( payload_id INTEGER, resource_item_id INTEGER, diff --git a/src/hydrilla/proxy/web_ui/_app.py b/src/hydrilla/proxy/web_ui/_app.py new file mode 100644 index 0000000..d5783d1 --- /dev/null +++ b/src/hydrilla/proxy/web_ui/_app.py @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: CC0-1.0 + +# Copyright (C) 2022 Wojtek Kosior +# +# Available under the terms of Creative Commons Zero v1.0 Universal. + +import flask + +from .. import state as st + + +class WebUIApp(flask.Flask): + _haketilo_state: st.HaketiloState diff --git a/src/hydrilla/proxy/web_ui/packages.py b/src/hydrilla/proxy/web_ui/packages.py index 7d67b63..a618ca0 100644 --- a/src/hydrilla/proxy/web_ui/packages.py +++ b/src/hydrilla/proxy/web_ui/packages.py @@ -31,14 +31,99 @@ # Enable using with Python 3.7. from __future__ import annotations +import tempfile +import zipfile import typing as t +from urllib.parse import urlparse +from pathlib import Path + import flask +import werkzeug + +from ...exceptions import HaketiloException +from ...translations import smart_gettext as _ +from .. import state as st +from . import _app + + +class InvalidUploadedMalcontent(HaketiloException): + def __init__(self): + super().__init__(_('err.proxy.uploaded_malcontent_invalid')) bp = flask.Blueprint('load_packages', __package__) -@bp.route('/packages/load_from_disk') -def load_from_disk() -> flask.Response: +@bp.route('/packages/load_from_disk', methods=['GET']) +def load_from_disk_get() -> flask.Response: html = flask.render_template('packages__load_from_disk.html.jinja') return flask.make_response(html, 200) + +@bp.route('/packages/load_from_disk', methods=['POST']) +def load_from_disk_post() -> werkzeug.Response: + parsed_url = urlparse(flask.request.referrer) + if parsed_url.netloc != 'hkt.mitm.it': + return load_from_disk_get() + + zip_file_storage = flask.request.files.get('packages_zipfile') + if zip_file_storage is None: + return load_from_disk_get() + + with tempfile.TemporaryDirectory() as tmpdir_str: + tmpdir = Path(tmpdir_str) + tmpdir_child = tmpdir / 'childdir' + tmpdir_child.mkdir() + + try: + with zipfile.ZipFile(zip_file_storage) as zip_file: + zip_file.extractall(tmpdir_child) + except: + raise HaketiloException(_('err.proxy.uploaded_file_not_zip')) + + extracted_top_level_files = tuple(tmpdir_child.iterdir()) + if extracted_top_level_files == (): + raise InvalidUploadedMalcontent() + + if len(extracted_top_level_files) == 1 and \ + extracted_top_level_files[0].is_dir(): + malcontent_dir_path = extracted_top_level_files[0] + else: + malcontent_dir_path = tmpdir_child + + state = t.cast(_app.WebUIApp, flask.current_app)._haketilo_state + + try: + state.import_packages(malcontent_dir_path) + except: + raise InvalidUploadedMalcontent() + + return flask.redirect(flask.url_for('.packages')) + +@bp.route('/packages') +def packages() -> flask.Response: + state = t.cast(_app.WebUIApp, flask.current_app)._haketilo_state + + display_infos = state.mapping_version_store().get_display_infos() + sorted_infos = sorted(display_infos, key=(lambda di: di.info)) + + html = flask.render_template( + 'packages.html.jinja', + display_infos = sorted_infos + ) + return flask.make_response(html, 200) + +@bp.route('/packages/view/') +def show_package(mapping_id: str) -> flask.Response: + state = t.cast(_app.WebUIApp, flask.current_app)._haketilo_state + + try: + store = state.mapping_version_store() + display_info = store.get(mapping_id).get_display_info() + + html = flask.render_template( + 'packages__show_single.html.jinja', + display_info = display_info + ) + return flask.make_response(html, 200) + except st.MissingItemError: + flask.abort(404) diff --git a/src/hydrilla/proxy/web_ui/root.py b/src/hydrilla/proxy/web_ui/root.py index 194251e..64d6be1 100644 --- a/src/hydrilla/proxy/web_ui/root.py +++ b/src/hydrilla/proxy/web_ui/root.py @@ -37,33 +37,36 @@ from threading import Lock import jinja2 import flask -from ...translations import smart_gettext as _ +from ...translations import translation as make_translation +from ... import versions from .. import state as st from .. import http_messages - from . import repos from . import packages +from . import _app -class WebUIApp(flask.Flask): +class WebUIAppImpl(_app.WebUIApp): def __init__(self): super().__init__(__name__) self.jinja_options = { **self.jinja_options, 'loader': jinja2.PackageLoader(__package__), - 'autoescape': jinja2.select_autoescape() + 'autoescape': jinja2.select_autoescape(['html.jinja']), + 'extensions': [ + *self.jinja_options.get('extensions', []), + 'jinja2.ext.i18n' + ] } for blueprint in [repos.bp, packages.bp]: self.register_blueprint(blueprint) - _haketilo_state: st.HaketiloState - # Flask app is not thread-safe and has to be accompanied by an ugly lock. This # can cause slow requests to block other requests, so we might need a better # workaround at some later point. -app = WebUIApp() +app = WebUIAppImpl() app_lock = Lock() @@ -83,6 +86,8 @@ def process_request( with app_lock: app._haketilo_state = state + app.jinja_env.install_gettext_translations(make_translation()) + flask_response = app.test_client().open( path = path, base_url = 'https://hkt.mitm.it', diff --git a/src/hydrilla/proxy/web_ui/templates/base.html.jinja b/src/hydrilla/proxy/web_ui/templates/base.html.jinja index c6f0dcf..4a9adf8 100644 --- a/src/hydrilla/proxy/web_ui/templates/base.html.jinja +++ b/src/hydrilla/proxy/web_ui/templates/base.html.jinja @@ -23,7 +23,19 @@ in a proprietary work, I am not going to enforce this in court. {% block head %} - {% block title required %}{% endblock %} - Haketilo proxy + {% block title required %}{% endblock %} - Haketilo proxy + {% endblock %} diff --git a/src/hydrilla/proxy/web_ui/templates/packages.html.jinja b/src/hydrilla/proxy/web_ui/templates/packages.html.jinja new file mode 100644 index 0000000..d0ba5cb --- /dev/null +++ b/src/hydrilla/proxy/web_ui/templates/packages.html.jinja @@ -0,0 +1,78 @@ +{# +Spdx-License-Identifier: GPL-3.0-or-later OR CC-BY-SA-4.0 + + +Proxy web UI package list page. + +This file is part of Hydrilla&Haketilo. + +Copyright (C) 2022 Wojtek Kosior + +Dual licensed under +* GNU General Public License v3.0 or later and +* Creative Commons Attribution Share Alike 4.0 International. + +You can choose to use either of these licenses or both. + + +I, Wojtek Kosior, thereby promise not to sue for violation of this +file's licenses. Although I request that you do not make use this code +in a proprietary work, I am not going to enforce this in court. +#} +{% extends "base.html.jinja" %} +{% block title %}Available packages{% endblock %} +{% block style %} +{{ super() }} + +ul#packages_list { + padding: 0; +} + +ul#packages_list > li { + list-style-type: none; + max-width: 100%; + overflow-x: scroll; + white-space: nowrap; + padding: 5px; + margin: 5px; + border: 2px solid #999; + border-radius: 5px; +} + +ul#packages_list > li > a { + display: block; + text-decoration: inherit; + color: inherit; +} + +.package-identifier { + font-size: 80%; + color: #555; +} + +{% endblock %} +{% block main %} +

{{ _('web_ui.h3.packages') }}

+ +{% endblock %} diff --git a/src/hydrilla/proxy/web_ui/templates/packages__load_from_disk.html.jinja b/src/hydrilla/proxy/web_ui/templates/packages__load_from_disk.html.jinja index 07ed3b3..52280b2 100644 --- a/src/hydrilla/proxy/web_ui/templates/packages__load_from_disk.html.jinja +++ b/src/hydrilla/proxy/web_ui/templates/packages__load_from_disk.html.jinja @@ -22,5 +22,17 @@ in a proprietary work, I am not going to enforce this in court. {% extends "base.html.jinja" %} {% block title %}Load package{% endblock %} {% block main %} - Not implemented yet :( +
+
+ +
+
+ +
+
+ +
+
{% endblock %} diff --git a/src/hydrilla/proxy/web_ui/templates/packages__show_single.html.jinja b/src/hydrilla/proxy/web_ui/templates/packages__show_single.html.jinja new file mode 100644 index 0000000..5e20dd7 --- /dev/null +++ b/src/hydrilla/proxy/web_ui/templates/packages__show_single.html.jinja @@ -0,0 +1,32 @@ +{# +Spdx-License-Identifier: GPL-3.0-or-later OR CC-BY-SA-4.0 + + +Proxy web UI package show page. + +This file is part of Hydrilla&Haketilo. + +Copyright (C) 2022 Wojtek Kosior + +Dual licensed under +* GNU General Public License v3.0 or later and +* Creative Commons Attribution Share Alike 4.0 International. + +You can choose to use either of these licenses or both. + + +I, Wojtek Kosior, thereby promise not to sue for violation of this +file's licenses. Although I request that you do not make use this code +in a proprietary work, I am not going to enforce this in court. +#} +{% extends "base.html.jinja" %} +{% block title %} Package details {% endblock %} +{% block style %} +{{ super() }} +{% endblock %} +{% block main %} +

{{ _('web_ui.h3.package_{}').format(display_info.info.long_name) }}

+
+ {{ display_info.info.versioned_identifier }} +
+{% endblock %} -- cgit v1.2.3