From 45e5cf8dc3ca936e2db8e7e45689d0a3331aad43 Mon Sep 17 00:00:00 2001 From: Wojtek Kosior Date: Thu, 8 Sep 2022 17:59:40 +0200 Subject: [proxy] make package auto-installation work --- src/hydrilla/proxy/addon.py | 9 ++ src/hydrilla/proxy/policies/payload.py | 45 +++++- src/hydrilla/proxy/policies/payload_resource.py | 2 +- src/hydrilla/proxy/state.py | 33 +++- .../proxy/state_impl/_operations/prune_orphans.py | 25 +++ src/hydrilla/proxy/state_impl/base.py | 3 + src/hydrilla/proxy/state_impl/concrete_state.py | 22 ++- src/hydrilla/proxy/state_impl/items.py | 2 +- src/hydrilla/proxy/state_impl/payloads.py | 170 ++++++++++++++++++++- src/hydrilla/proxy/web_ui/items.py | 6 + src/hydrilla/proxy/web_ui/prompts.py | 115 ++++++++++++++ src/hydrilla/proxy/web_ui/root.py | 3 +- .../prompts/auto_install_error.html.jinja | 60 ++++++++ 13 files changed, 475 insertions(+), 20 deletions(-) create mode 100644 src/hydrilla/proxy/web_ui/prompts.py create mode 100644 src/hydrilla/proxy/web_ui/templates/prompts/auto_install_error.html.jinja (limited to 'src/hydrilla/proxy') diff --git a/src/hydrilla/proxy/addon.py b/src/hydrilla/proxy/addon.py index c7fae7e..b9c4159 100644 --- a/src/hydrilla/proxy/addon.py +++ b/src/hydrilla/proxy/addon.py @@ -40,6 +40,7 @@ import traceback as tb from threading import Lock from pathlib import Path from contextlib import contextmanager +from urllib.parse import urlparse from mitmproxy import tls, http, addonmanager, ctx from mitmproxy.script import concurrent @@ -184,6 +185,14 @@ class HaketiloAddon: available at this point, so the handler effectively does nothing. """ with self.http_safe_event_handling(flow): + referrer = flow.request.headers.get('referer') + if referrer is not None: + if urlparse(referrer).netloc == 'hkt.mitm.it' and \ + urlparse(flow.request.url).netloc != 'hkt.mitm.it': + # Do not reveal to the site that Haketilo meta-site was + # visited before. + flow.request.headers.pop('referer', None) + policy = self.try_get_policy(flow) if policy is not None: diff --git a/src/hydrilla/proxy/policies/payload.py b/src/hydrilla/proxy/policies/payload.py index e64653d..a063c7c 100644 --- a/src/hydrilla/proxy/policies/payload.py +++ b/src/hydrilla/proxy/policies/payload.py @@ -35,8 +35,12 @@ import dataclasses as dc import typing as t import re +from urllib.parse import urlencode + +from itsdangerous.url_safe import URLSafeSerializer import bs4 # type: ignore +from ...exceptions import HaketiloException from ...url_patterns import ParsedUrl from .. import csp from .. import state @@ -65,7 +69,7 @@ class PayloadAwarePolicyFactory(base.PolicyFactory): @property def payload_ref(self) -> state.PayloadRef: """....""" - return self.payload_key.payload_ref + return self.payload_key.ref def __lt__(self, other: base.PolicyFactory) -> bool: """....""" @@ -171,7 +175,7 @@ class PayloadInjectPolicy(PayloadAwarePolicy): def _script_urls(self, url: ParsedUrl) -> t.Iterable[str]: """....""" base_url = self.assets_base_url(url) - payload_ref = self.payload_data.payload_ref + payload_ref = self.payload_data.ref for path in payload_ref.get_script_paths(): yield base_url + '/'.join(('static', *path)) @@ -261,10 +265,47 @@ class PayloadInjectPolicy(PayloadAwarePolicy): ) +class _PayloadHasProblemsError(HaketiloException): + pass + class AutoPayloadInjectPolicy(PayloadInjectPolicy): """....""" priority: t.ClassVar[base.PolicyPriority] = base.PolicyPriority._ONE + def consume_response(self, response_info: http_messages.ResponseInfo) \ + -> http_messages.ProducedResponse: + try: + if self.payload_data.ref.has_problems(): + raise _PayloadHasProblemsError() + + self.payload_data.ref.ensure_items_installed() + + return super().consume_response(response_info) + except (state.RepoCommunicationError, state.FileInstallationError, + _PayloadHasProblemsError) as ex: + params = { + 'next_url': response_info.url.orig_url, + 'payload_id': self.payload_data.ref.id + } + + if isinstance(ex, state.FileInstallationError): + params['repo_id'] = ex.repo_id + params['file_sha256'] = ex.sha256 + + serializer = URLSafeSerializer( + self.payload_data.global_secret, + salt = 'auto_install_error' + ) + query = urlencode({'details': serializer.dumps(params)}) + redirect_url = 'https://hkt.mitm.it/auto_install_error?' + query + msg = 'Error occured when installing payload. Redirecting.' + + return http_messages.ProducedResponse( + status_code = 303, + headers = [(b'Location', redirect_url.encode())], + body = msg.encode() + ) + @dc.dataclass(frozen=True) class PayloadSuggestPolicy(PayloadAwarePolicy): diff --git a/src/hydrilla/proxy/policies/payload_resource.py b/src/hydrilla/proxy/policies/payload_resource.py index 3aacea0..f50f574 100644 --- a/src/hydrilla/proxy/policies/payload_resource.py +++ b/src/hydrilla/proxy/policies/payload_resource.py @@ -77,7 +77,7 @@ class PayloadResourcePolicy(PayloadAwarePolicy): -> http_messages.ProducedResponse: """....""" try: - file_data = self.payload_data.payload_ref.get_file_data(path) + file_data = self.payload_data.ref.get_file_data(path) except state.MissingItemError: return resource_blocked_response diff --git a/src/hydrilla/proxy/state.py b/src/hydrilla/proxy/state.py index 03b6291..491a865 100644 --- a/src/hydrilla/proxy/state.py +++ b/src/hydrilla/proxy/state.py @@ -337,7 +337,7 @@ class ResourceVersionStore(Store[ResourceVersionRef]): @dc.dataclass(frozen=True) class PayloadKey: """....""" - payload_ref: 'PayloadRef' + ref: 'PayloadRef' mapping_identifier: str @@ -348,13 +348,22 @@ class PayloadKey: @dc.dataclass(frozen=True) class PayloadData: """....""" - payload_ref: 'PayloadRef' + ref: 'PayloadRef' explicitly_enabled: bool unique_token: str pattern_path_segments: tuple[str, ...] eval_allowed: bool cors_bypass_allowed: bool + global_secret: bytes + +@dc.dataclass(frozen=True) +class PayloadDisplayInfo: + ref: 'PayloadRef' + + mapping_info: MappingVersionDisplayInfo + pattern: str + has_problems: bool @dc.dataclass(frozen=True) class FileData: @@ -371,7 +380,15 @@ class PayloadRef(Ref): ... @abstractmethod - def get_mapping(self) -> MappingVersionRef: + def has_problems(self) -> bool: + ... + + @abstractmethod + def get_display_info(self) -> PayloadDisplayInfo: + ... + + @abstractmethod + def ensure_items_installed(self) -> None: """....""" ... @@ -387,6 +404,9 @@ class PayloadRef(Ref): """....""" ... +class PayloadStore(Store[PayloadRef]): + pass + class MappingUseMode(Enum): """ @@ -449,8 +469,11 @@ class HaketiloState(ABC): ... @abstractmethod - def get_payload(self, payload_id: str) -> PayloadRef: - """....""" + def payload_store(self) -> PayloadStore: + ... + + @abstractmethod + def get_secret(self) -> bytes: ... @abstractmethod diff --git a/src/hydrilla/proxy/state_impl/_operations/prune_orphans.py b/src/hydrilla/proxy/state_impl/_operations/prune_orphans.py index f4ebd52..5a3f4f0 100644 --- a/src/hydrilla/proxy/state_impl/_operations/prune_orphans.py +++ b/src/hydrilla/proxy/state_impl/_operations/prune_orphans.py @@ -102,6 +102,30 @@ WHERE file_id IN removed_files; ''' +_forget_files_data_sql = ''' +WITH forgotten_files AS ( + SELECT + f.file_id + FROM + files AS f + JOIN file_uses AS fu + USING (file_id) + LEFT JOIN item_versions AS iv + ON (fu.item_version_id = iv.item_version_id AND + iv.installed = 'I') + GROUP BY + f.file_id + HAVING + COUNT(iv.item_version_id) = 0 +) +UPDATE + files +SET + data = NULL +WHERE + file_id IN forgotten_files; +''' + _remove_repo_iterations_sql = ''' WITH removed_iterations AS ( SELECT @@ -141,5 +165,6 @@ def prune_orphans(cursor: sqlite3.Cursor) -> None: cursor.execute(sql) cursor.execute(_remove_items_sql) cursor.execute(_remove_files_sql) + cursor.execute(_forget_files_data_sql) cursor.execute(_remove_repo_iterations_sql) cursor.execute(_remove_repos_sql) diff --git a/src/hydrilla/proxy/state_impl/base.py b/src/hydrilla/proxy/state_impl/base.py index 82b8734..0559a42 100644 --- a/src/hydrilla/proxy/state_impl/base.py +++ b/src/hydrilla/proxy/state_impl/base.py @@ -35,6 +35,7 @@ from __future__ import annotations import sqlite3 import threading +import secrets import dataclasses as dc import typing as t @@ -143,6 +144,8 @@ class HaketiloStateWithFields(st.HaketiloState): #settings: st.HaketiloGlobalSettings + secret: bytes = dc.field(default_factory=(lambda: secrets.token_bytes(16))) + policy_tree: PolicyTree = PolicyTree() payloads_data: PayloadsData = dc.field(default_factory=dict) diff --git a/src/hydrilla/proxy/state_impl/concrete_state.py b/src/hydrilla/proxy/state_impl/concrete_state.py index 4781baa..8bd25a9 100644 --- a/src/hydrilla/proxy/state_impl/concrete_state.py +++ b/src/hydrilla/proxy/state_impl/concrete_state.py @@ -158,12 +158,14 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields): _operations.pull_missing_files(cursor) def rebuild_structures(self) -> None: - with self.cursor(transaction=True) as cursor: + with self.cursor() as cursor: cursor.execute( ''' SELECT - p.payload_id, p.pattern, p.eval_allowed, - p.cors_bypass_allowed, + p.payload_id, + p.pattern, + p.eval_allowed, + p.cors_bypass_allowed, ms.enabled, i.identifier FROM @@ -175,7 +177,7 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields): ''' ) - rows = cursor.fetchall() + rows = cursor.fetchall() new_policy_tree = base.PolicyTree() @@ -214,12 +216,13 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields): pattern_path_segments = parsed_pattern.path_segments payload_data = st.PayloadData( - payload_ref = payload_ref, + ref = payload_ref, explicitly_enabled = enabled_status == 'E', unique_token = token, pattern_path_segments = pattern_path_segments, eval_allowed = eval_allowed, - cors_bypass_allowed = cors_bypass_allowed + cors_bypass_allowed = cors_bypass_allowed, + global_secret = self.secret ) new_payloads_data[payload_ref] = payload_data @@ -245,8 +248,11 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields): def resource_version_store(self) -> st.ResourceVersionStore: return items.ConcreteResourceVersionStore(self) - def get_payload(self, payload_id: str) -> st.PayloadRef: - raise NotImplementedError() + def payload_store(self) -> st.PayloadStore: + return payloads.ConcretePayloadStore(self) + + def get_secret(self) -> bytes: + return self.secret def get_settings(self) -> st.HaketiloGlobalSettings: return st.HaketiloGlobalSettings( diff --git a/src/hydrilla/proxy/state_impl/items.py b/src/hydrilla/proxy/state_impl/items.py index 5f2f274..3ba8f80 100644 --- a/src/hydrilla/proxy/state_impl/items.py +++ b/src/hydrilla/proxy/state_impl/items.py @@ -232,7 +232,7 @@ class ConcreteMappingRef(st.MappingRef): new_frozen_status == old_frozen_status): return else: - if old_active_version_id is None: + if old_active_version_id is None and old_enabled_status != 'D': return self.state.recompute_dependencies([int(self.id)]) diff --git a/src/hydrilla/proxy/state_impl/payloads.py b/src/hydrilla/proxy/state_impl/payloads.py index 2bee11f..e622e52 100644 --- a/src/hydrilla/proxy/state_impl/payloads.py +++ b/src/hydrilla/proxy/state_impl/payloads.py @@ -31,11 +31,14 @@ This module provides an interface to interact with payloads inside Haketilo. # Enable using with Python 3.7. from __future__ import annotations +import sqlite3 import dataclasses as dc import typing as t +from ... import item_infos from .. import state as st from . import base +from . import items @dc.dataclass(frozen=True, unsafe_hash=True) @@ -48,8 +51,163 @@ class ConcretePayloadRef(st.PayloadRef): except KeyError: raise st.MissingItemError() - def get_mapping(self) -> st.MappingVersionRef: - raise NotImplementedError() + def has_problems(self) -> bool: + with self.state.cursor(transaction=True) as cursor: + cursor.execute( + ''' + SELECT + iv.installed == 'F' + FROM + payloads AS p + JOIN item_versions AS iv + ON p.mapping_item_id = iv.item_version_id + WHERE + p.payload_id = ?; + ''', + (self.id,) + ) + + rows = cursor.fetchall() + + if rows == []: + raise st.MissingItemError() + + (mapping_install_failed,), = rows + if mapping_install_failed: + return True + + cursor.execute( + ''' + SELECT + COUNT(*) > 0 + FROM + payloads AS p + JOIN resolved_depended_resources AS rdd + USING (payload_id) + JOIN item_versions AS iv + ON rdd.resource_item_id = iv.item_version_id + WHERE + p.payload_id = ? AND iv.installed = 'F'; + ''', + (self.id,) + ) + + (resource_install_failed,), = cursor.fetchall() + if resource_install_failed: + return True + + return False + + def get_display_info(self) -> st.PayloadDisplayInfo: + with self.state.cursor() as cursor: + cursor.execute( + ''' + SELECT + p.pattern, + ive.item_version_id, + ive.definition, + ive.repo, + ive.repo_iteration, + ive.installed, + ive.active, + ive.is_orphan, + ive.is_local + FROM + payloads AS p + JOIN item_versions_extra AS ive + ON p.mapping_item_id = ive.item_version_id + WHERE + p.payload_id = ?; + ''', + (self.id,) + ) + + rows = cursor.fetchall() + + if rows == []: + raise st.MissingItemError() + + (pattern_str, mapping_version_id, definition, repo, repo_iteration, + installed_status, active_status, is_orphan, is_local), = rows + + has_problems = self.has_problems() + + mapping_version_ref = items.ConcreteMappingVersionRef( + id = str(mapping_version_id), + state = self.state + ) + + mapping_version_info = item_infos.MappingInfo.load( + definition, + repo, + repo_iteration + ) + + mapping_version_display_info = st.MappingVersionDisplayInfo( + ref = mapping_version_ref, + info = mapping_version_info, + installed = st.InstalledStatus(installed_status), + active = st.ActiveStatus(active_status), + is_orphan = is_orphan, + is_local = is_local + ) + + return st.PayloadDisplayInfo( + ref = self, + mapping_info = mapping_version_display_info, + pattern = pattern_str, + has_problems = has_problems + ) + + def ensure_items_installed(self) -> None: + with self.state.cursor(transaction=True) as cursor: + cursor.execute( + 'SELECT mapping_item_id FROM payloads WHERE payload_id = ?;', + (self.id,) + ) + + rows = cursor.fetchall() + + if rows == []: + raise st.MissingItemError() + + (mapping_version_id,), = rows + + cursor.execute( + ''' + UPDATE + item_versions + SET + installed = 'I' + WHERE + item_version_id = ?; + ''', + (mapping_version_id,) + ) + + cursor.execute( + ''' + WITH depended_resource_ids AS ( + SELECT + rdd.resource_item_id + FROM + payloads AS p + JOIN resolved_depended_resources AS rdd + USING (payload_id) + WHERE + payload_id = ? + ) + UPDATE + item_versions + SET + installed = 'I' + WHERE + item_version_id IN depended_resource_ids; + ''', + (self.id,) + ) + + self.state.pull_missing_files() def get_script_paths(self) \ -> t.Iterable[t.Sequence[str]]: @@ -135,3 +293,11 @@ class ConcretePayloadRef(st.PayloadRef): (data, mime_type), = result return st.FileData(type=mime_type, name=file_name, contents=data) + + +@dc.dataclass(frozen=True) +class ConcretePayloadStore(st.PayloadStore): + state: base.HaketiloStateWithFields + + def get(self, id: str) -> st.PayloadRef: + return ConcretePayloadRef(str(int(id)), self.state) diff --git a/src/hydrilla/proxy/web_ui/items.py b/src/hydrilla/proxy/web_ui/items.py index 1a56d7d..a7e497b 100644 --- a/src/hydrilla/proxy/web_ui/items.py +++ b/src/hydrilla/proxy/web_ui/items.py @@ -238,6 +238,12 @@ def alter_item_version(item_version_id: str, item_type: item_infos.ItemType) \ return show_item_version(item_version_id, item_type) else: raise ValueError() + except st.RepoCommunicationError: + return show_item_version( + item_version_id = item_version_id, + item_type = item_type, + errors = {'repo_communication_error': True} + ) except st.FileInstallationError: return show_item_version( item_version_id = item_version_id, diff --git a/src/hydrilla/proxy/web_ui/prompts.py b/src/hydrilla/proxy/web_ui/prompts.py new file mode 100644 index 0000000..b546e47 --- /dev/null +++ b/src/hydrilla/proxy/web_ui/prompts.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +# Proxy web UI pages that may be shown to the user without manual navigation to +# Haketilo meta-site. +# +# This file is part of Hydrilla&Haketilo. +# +# Copyright (C) 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this +# file's license. Although I request that you do not make use this code +# in a proprietary program, I am not going to enforce this in court. + +# Enable using with Python 3.7. +from __future__ import annotations + +import typing as t + +from urllib.parse import urlencode + +from itsdangerous.url_safe import URLSafeSerializer +import flask +import werkzeug + +from .. import state as st +from . import _app + + +bp = flask.Blueprint('prompts', __package__) + +@bp.route('/auto_install_error', methods=['GET']) +def auto_install_error_prompt() -> werkzeug.Response: + state = _app.get_haketilo_state() + + serializer = URLSafeSerializer( + state.get_secret(), + salt = 'auto_install_error' + ) + + try: + details: t.Mapping[str, str] = \ + serializer.loads(flask.request.args['details']) + except: + return flask.redirect(flask.url_for('home')) + + try: + payload_ref = state.payload_store().get(details['payload_id']) + + display_info = payload_ref.get_display_info() + + html = flask.render_template( + 'prompts/auto_install_error.html.jinja', + display_info = display_info, + next_url = details['next_url'] + ) + return flask.make_response(html, 200) + except st.MissingItemError: + flask.abort(404) + +@bp.route('/auto_install_error', methods=['POST']) +def auto_install_error_prompt_post() -> werkzeug.Response: + form_data = flask.request.form + action = form_data['action'] + + mapping_ver_id = str(int(form_data['mapping_ver_id'])) + payload_id = str(int(form_data['payload_id'])) + next_url = form_data['next_url'] + + state = _app.get_haketilo_state() + + try: + mapping_ver_store = state.mapping_version_store() + mapping_ver_ref = mapping_ver_store.get(mapping_ver_id) + + payload_store = _app.get_haketilo_state().payload_store() + payload_ref = payload_store.get(payload_id) + + if action == 'disable_mapping': + mapping_ver_ref.update_mapping_status(st.EnabledStatus.DISABLED) + elif action == 'retry_install': + payload_ref.ensure_items_installed() + else: + raise ValueError() + except (st.RepoCommunicationError, st.FileInstallationError): + params = {'payload_id': payload_id, 'next_url': next_url} + + serializer = URLSafeSerializer( + state.get_secret(), + salt = 'auto_install_error' + ) + query = urlencode({'details': params}) + redirect_url = flask.url_for( + '.auto_install_error_prompt', + details = serializer.dumps(params) + ) + + return flask.redirect(redirect_url) + except st.MissingItemError: + flask.abort(404) + + return flask.redirect(next_url) diff --git a/src/hydrilla/proxy/web_ui/root.py b/src/hydrilla/proxy/web_ui/root.py index eac3be7..ff7c1f7 100644 --- a/src/hydrilla/proxy/web_ui/root.py +++ b/src/hydrilla/proxy/web_ui/root.py @@ -46,6 +46,7 @@ from .. import state as st from .. import http_messages from . import repos from . import items +from . import prompts from . import _app @@ -91,7 +92,7 @@ class WebUIAppImpl(_app.WebUIApp): self.before_request(authenticate_by_referrer) - for blueprint in [repos.bp, items.bp]: + for blueprint in [repos.bp, items.bp, prompts.bp]: self.register_blueprint(blueprint) # Flask app is not thread-safe and has to be accompanied by an ugly lock. This diff --git a/src/hydrilla/proxy/web_ui/templates/prompts/auto_install_error.html.jinja b/src/hydrilla/proxy/web_ui/templates/prompts/auto_install_error.html.jinja new file mode 100644 index 0000000..01f5c19 --- /dev/null +++ b/src/hydrilla/proxy/web_ui/templates/prompts/auto_install_error.html.jinja @@ -0,0 +1,60 @@ +{# +SPDX-License-Identifier: GPL-3.0-or-later OR CC-BY-SA-4.0 + +Proxy web UI page that informs about failure of automatic package installation. + +This file is part of Hydrilla&Haketilo. + +Copyright (C) 2022 Wojtek Kosior + +Dual licensed under +* GNU General Public License v3.0 or later and +* Creative Commons Attribution Share Alike 4.0 International. + +You can choose to use either of these licenses or both. + + +I, Wojtek Kosior, thereby promise not to sue for violation of this +file's licenses. Although I request that you do not make use this code +in a proprietary work, I am not going to enforce this in court. +#} +{% extends "base.html.jinja" %} + +{% block title %} + {{ _('web_ui.prompts.auto_install_error.title') }} +{% endblock %} + +{% macro button_form(action, button_class, button_text) %} +
+ + + {% set mapping_ver_id = display_info.mapping_info.ref.id %} + + + + +
+{% endmacro %} + +{% block main %} +

+ {{ _('web_ui.prompts.auto_install_error.heading') }} +

+ +

+ {{ + _('web_ui.prompts.auto_install_error.package_{}_failed_to_install') + .format(display_info.mapping_info.info.long_name) + }} +

+ +
+ {% set but_text = _('web_ui.prompts.auto_install_error.disable_button') %} + {{ button_form('disable_mapping', 'red-button', but_text) }} + +
+ + {% set but_text = _('web_ui.prompts.auto_install_error.retry_button') %} + {{ button_form('retry_install', 'green-button', but_text) }} +
+{% endblock %} -- cgit v1.2.3