From ffe6e681b281eef2c1e62bbc6feefb0b85d040ca Mon Sep 17 00:00:00 2001 From: Wojtek Kosior Date: Wed, 28 Sep 2022 12:48:45 +0200 Subject: [proxy] make it possible to create "ad hoc" packages through the web UI --- src/hydrilla/locales/en_US/LC_MESSAGES/messages.po | 142 ++++++++++----- src/hydrilla/proxy/web_ui/items.py | 47 +---- src/hydrilla/proxy/web_ui/items_import.py | 196 +++++++++++++++++++++ src/hydrilla/proxy/web_ui/root.py | 5 +- .../proxy/web_ui/templates/base.html.jinja | 36 +++- .../proxy/web_ui/templates/import.html.jinja | 120 +++++++++++++ .../templates/items/load_from_disk.html.jinja | 68 ------- .../proxy/web_ui/templates/repos/add.html.jinja | 25 +-- 8 files changed, 459 insertions(+), 180 deletions(-) create mode 100644 src/hydrilla/proxy/web_ui/items_import.py create mode 100644 src/hydrilla/proxy/web_ui/templates/import.html.jinja delete mode 100644 src/hydrilla/proxy/web_ui/templates/items/load_from_disk.html.jinja diff --git a/src/hydrilla/locales/en_US/LC_MESSAGES/messages.po b/src/hydrilla/locales/en_US/LC_MESSAGES/messages.po index e33c5e7..28f6791 100644 --- a/src/hydrilla/locales/en_US/LC_MESSAGES/messages.po +++ b/src/hydrilla/locales/en_US/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: hydrilla 2.0\n" "Report-Msgid-Bugs-To: koszko@koszko.org\n" -"POT-Creation-Date: 2022-09-27 13:36+0200\n" +"POT-Creation-Date: 2022-09-28 10:27+0200\n" "PO-Revision-Date: 2022-02-12 00:00+0000\n" "Last-Translator: Wojtek Kosior \n" "Language: en_US\n" @@ -217,33 +217,113 @@ msgstr "" "This installation of Haketilo uses an SQLite version which does not " "support foreign key constraints." -#: src/hydrilla/proxy/web_ui/templates/base.html.jinja:71 +#: src/hydrilla/proxy/web_ui/templates/base.html.jinja:101 msgid "web_ui.base.title.haketilo_proxy" msgstr "Haketilo" -#: src/hydrilla/proxy/web_ui/templates/base.html.jinja:241 +#: src/hydrilla/proxy/web_ui/templates/base.html.jinja:275 msgid "web_ui.base.nav.home" msgstr "Home" -#: src/hydrilla/proxy/web_ui/templates/base.html.jinja:242 +#: src/hydrilla/proxy/web_ui/templates/base.html.jinja:276 msgid "web_ui.base.nav.rules" msgstr "Script blocking" -#: src/hydrilla/proxy/web_ui/templates/base.html.jinja:243 +#: src/hydrilla/proxy/web_ui/templates/base.html.jinja:277 msgid "web_ui.base.nav.packages" msgstr "Packages" -#: src/hydrilla/proxy/web_ui/templates/base.html.jinja:244 +#: src/hydrilla/proxy/web_ui/templates/base.html.jinja:278 msgid "web_ui.base.nav.libraries" msgstr "Libraries" -#: src/hydrilla/proxy/web_ui/templates/base.html.jinja:245 +#: src/hydrilla/proxy/web_ui/templates/base.html.jinja:279 msgid "web_ui.base.nav.repos" msgstr "Repositories" -#: src/hydrilla/proxy/web_ui/templates/base.html.jinja:246 -msgid "web_ui.base.nav.load" -msgstr "Import from file" +#: src/hydrilla/proxy/web_ui/templates/base.html.jinja:280 +msgid "web_ui.base.nav.import" +msgstr "Import" + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:23 +msgid "web_ui.import.title" +msgstr "Import items" + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:41 +msgid "web_ui.import.heading" +msgstr "Import items" + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:43 +msgid "web_ui.import.heading_import_from_file" +msgstr "From file" + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:49 +msgid "web_ui.err.uploaded_file_not_zip" +msgstr "The uploaded file is not a valid ZIP file." + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:53 +msgid "web_ui.err.invalid_uploaded_malcontent" +msgstr "The uploaded archive does not contain valid Haketilo malcontent." + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:61 +msgid "web_ui.import.choose_zipfile_button" +msgstr "Select file" + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:68 +msgid "web_ui.import.install_from_file_button" +msgstr "Import" + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:75 +msgid "web_ui.import.heading_import_ad_hoc" +msgstr "Ad hoc" + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:78 +msgid "web_ui.err.invalid_ad_hoc_package" +msgstr "The ad hoc package being imported contains errors." + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:84 +msgid "web_ui.import.identifier_field_label" +msgstr "Identifier" + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:86 +msgid "web_ui.err.invalid_ad_hoc_identifier" +msgstr "Chosen identifier is not valid." + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:90 +msgid "web_ui.import.long_name_field_label" +msgstr "Long name (optional)" + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:93 +msgid "web_ui.import.version_field_label" +msgstr "Version (optional)" + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:95 +msgid "web_ui.err.invalid_ad_hoc_version" +msgstr "Chosen version is not valid." + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:99 +msgid "web_ui.import.description_field_label" +msgstr "Description (optional)" + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:102 +msgid "web_ui.import.patterns_field_label" +msgstr "URL patterns (each on its own line)" + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:104 +msgid "web_ui.err.invalid_ad_hoc_patterns" +msgstr "Chosen patterns are not valid." + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:108 +msgid "web_ui.import.script_text_field_label" +msgstr "JavaScript to execute on pages that match one of the patterns" + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:111 +msgid "web_ui.import.lic_text_field_label" +msgstr "Package license text (optional)" + +#: src/hydrilla/proxy/web_ui/templates/import.html.jinja:116 +msgid "web_ui.import.install_ad_hoc_button" +msgstr "Add new package" #: src/hydrilla/proxy/web_ui/templates/index.html.jinja:23 msgid "web_ui.home.title" @@ -454,30 +534,6 @@ msgstr "" "explicitly configured by the user but is going to be activated " "automatically." -#: src/hydrilla/proxy/web_ui/templates/items/load_from_disk.html.jinja:23 -msgid "web_ui.load_from_disk.title" -msgstr "Import from file" - -#: src/hydrilla/proxy/web_ui/templates/items/load_from_disk.html.jinja:41 -msgid "web_ui.load_from_disk.heading" -msgstr "Import items from a ZIP archive" - -#: src/hydrilla/proxy/web_ui/templates/items/load_from_disk.html.jinja:45 -msgid "web_ui.err.uploaded_file_not_zip" -msgstr "The uploaded file is not a valid ZIP file." - -#: src/hydrilla/proxy/web_ui/templates/items/load_from_disk.html.jinja:49 -msgid "web_ui.err.invalid_uploaded_malcontent" -msgstr "The uploaded archive does not contain valid Haketilo malcontent." - -#: src/hydrilla/proxy/web_ui/templates/items/load_from_disk.html.jinja:57 -msgid "web_ui.load_from_disk.choose_zipfile_button" -msgstr "Select file" - -#: src/hydrilla/proxy/web_ui/templates/items/load_from_disk.html.jinja:64 -msgid "web_ui.load_from_disk.install_button" -msgstr "Import" - #: src/hydrilla/proxy/web_ui/templates/items/package_view.html.jinja:23 msgid "web_ui.items.single.package.title" msgstr "Package view" @@ -676,7 +732,11 @@ msgstr "New repository" msgid "web_ui.repos.add.heading" msgstr "Configure a new repository" -#: src/hydrilla/proxy/web_ui/templates/repos/add.html.jinja:29 +#: src/hydrilla/proxy/web_ui/templates/repos/add.html.jinja:28 +msgid "web_ui.repos.add.name_field_label" +msgstr "Name" + +#: src/hydrilla/proxy/web_ui/templates/repos/add.html.jinja:30 #: src/hydrilla/proxy/web_ui/templates/repos/show_single.html.jinja:67 msgid "web_ui.err.repo_name_invalid" msgstr "Chosen name is not valid." @@ -687,19 +747,15 @@ msgid "web_ui.err.repo_name_taken" msgstr "Chosen name is already in use." #: src/hydrilla/proxy/web_ui/templates/repos/add.html.jinja:37 -msgid "web_ui.repos.add.name_field_label" -msgstr "Name" +msgid "web_ui.repos.add.url_field_label" +msgstr "URL" -#: src/hydrilla/proxy/web_ui/templates/repos/add.html.jinja:47 +#: src/hydrilla/proxy/web_ui/templates/repos/add.html.jinja:39 #: src/hydrilla/proxy/web_ui/templates/repos/show_single.html.jinja:121 msgid "web_ui.err.repo_url_invalid" msgstr "Chosen URL is not vald." -#: src/hydrilla/proxy/web_ui/templates/repos/add.html.jinja:51 -msgid "web_ui.repos.add.url_field_label" -msgstr "URL" - -#: src/hydrilla/proxy/web_ui/templates/repos/add.html.jinja:62 +#: src/hydrilla/proxy/web_ui/templates/repos/add.html.jinja:45 msgid "web_ui.repos.add.submit_button" msgstr "Add repository" diff --git a/src/hydrilla/proxy/web_ui/items.py b/src/hydrilla/proxy/web_ui/items.py index 6195107..f34b89b 100644 --- a/src/hydrilla/proxy/web_ui/items.py +++ b/src/hydrilla/proxy/web_ui/items.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later -# Proxy web UI packages loading. +# Proxy web UI package/library management. # # This file is part of Hydrilla&Haketilo. # @@ -32,17 +32,11 @@ # Enable using with Python 3.7. from __future__ import annotations -import tempfile -import zipfile import typing as t -from pathlib import Path - import flask import werkzeug -from ...exceptions import HaketiloException -from ...translations import smart_gettext as _ from ... import item_infos from .. import state as st from . import _app @@ -50,45 +44,6 @@ from . import _app bp = flask.Blueprint('items', __package__) -@bp.route('/load_from_disk', methods=['GET']) -def load_from_disk(errors: t.Mapping[str, bool] = {}) -> werkzeug.Response: - html = flask.render_template('items/load_from_disk.html.jinja', **errors) - return flask.make_response(html, 200) - -@bp.route('/load_from_disk', methods=['POST']) -def load_from_disk_post() -> werkzeug.Response: - zip_file_storage = flask.request.files.get('items_zipfile') - if zip_file_storage is None: - return load_from_disk() - - with tempfile.TemporaryDirectory() as tmpdir_str: - tmpdir = Path(tmpdir_str) - tmpdir_child = tmpdir / 'childdir' - tmpdir_child.mkdir() - - try: - with zipfile.ZipFile(zip_file_storage) as zip_file: - zip_file.extractall(tmpdir_child) - except: - return load_from_disk({'uploaded_file_not_zip': True}) - - extracted_top_level_files = tuple(tmpdir_child.iterdir()) - if extracted_top_level_files == (): - return load_from_disk({'invalid_uploaded_malcontent': True}) - - if len(extracted_top_level_files) == 1 and \ - extracted_top_level_files[0].is_dir(): - malcontent_dir_path = extracted_top_level_files[0] - else: - malcontent_dir_path = tmpdir_child - - try: - _app.get_haketilo_state().import_items(malcontent_dir_path) - except: - return load_from_disk({'invalid_uploaded_malcontent': True}) - - return flask.redirect(flask.url_for('.packages')) - @bp.route('/packages') def packages() -> werkzeug.Response: store = _app.get_haketilo_state().mapping_store() diff --git a/src/hydrilla/proxy/web_ui/items_import.py b/src/hydrilla/proxy/web_ui/items_import.py new file mode 100644 index 0000000..b335237 --- /dev/null +++ b/src/hydrilla/proxy/web_ui/items_import.py @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +# Proxy web UI packages loading. +# +# This file is part of Hydrilla&Haketilo. +# +# Copyright (C) 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this +# file's license. Although I request that you do not make use of this +# code in a proprietary program, I am not going to enforce this in +# court. + +""" +..... +""" + +# Enable using with Python 3.7. +from __future__ import annotations + +import tempfile +import zipfile +import re +import json +import typing as t + +from pathlib import Path + +import flask +import werkzeug + +from ...url_patterns import normalize_pattern +from ...builder import build +from ... import versions +from .. import state as st +from . import _app + + +bp = flask.Blueprint('import', __package__) + +@bp.route('/import', methods=['GET']) +def items_import(errors: t.Mapping[str, bool] = {}) -> werkzeug.Response: + html = flask.render_template('import.html.jinja', **errors) + return flask.make_response(html, 200) + +def items_import_from_file() -> werkzeug.Response: + zip_file_storage = flask.request.files.get('items_zipfile') + if zip_file_storage is None: + return items_import() + + with tempfile.TemporaryDirectory() as tmpdir_str: + tmpdir = Path(tmpdir_str) + tmpdir_child = tmpdir / 'childdir' + tmpdir_child.mkdir() + + try: + with zipfile.ZipFile(zip_file_storage) as zip_file: + zip_file.extractall(tmpdir_child) + except: + return items_import({'uploaded_file_not_zip': True}) + + extracted_top_level_files = tuple(tmpdir_child.iterdir()) + if extracted_top_level_files == (): + return items_import({'invalid_uploaded_malcontent': True}) + + if len(extracted_top_level_files) == 1 and \ + extracted_top_level_files[0].is_dir(): + malcontent_dir_path = extracted_top_level_files[0] + else: + malcontent_dir_path = tmpdir_child + + try: + _app.get_haketilo_state().import_items(malcontent_dir_path) + except: + return items_import({'invalid_uploaded_malcontent': True}) + + return flask.redirect(flask.url_for('items.packages')) + +identifier_re = re.compile(r'^[-0-9a-z.]+$') + +def item_import_ad_hoc() -> werkzeug.Response: + form = flask.request.form + def get_as_str(field_name: str) -> str: + value = form[field_name] + assert isinstance(value, str) + return value.strip() + + try: + identifier = get_as_str('identifier') + assert identifier + assert identifier_re.match(identifier) + except: + return items_import({'invalid_ad_hoc_identifier': True}) + + long_name = get_as_str('long_name') or identifier + + resource_ref = {'identifier': identifier} + + try: + ver = versions.parse(get_as_str('version') or '1') + except: + return items_import({'invalid_ad_hoc_version': True}) + + try: + pat_str = get_as_str('patterns') + patterns = [ + normalize_pattern(p.strip()) + for p in pat_str.split('\n') + if p and not p.isspace() + ] + assert patterns + except: + return items_import({'invalid_ad_hoc_patterns': True}) + + common_definition_fields: t.Mapping[str, t.Any] = { + 'identifier': identifier, + 'long_name': long_name, + 'version': ver, + 'description': get_as_str('description') + } + + schema_url = \ + 'https://hydrilla.koszko.org/schemas/package_source-1.schema.json' + + package_index_json = { + '$schema': schema_url, + 'source_name': 'haketilo-ad-hoc-package', + 'copyright': [], + 'upstream_url': '', + 'definitions': [{ + **common_definition_fields, + 'type': 'mapping', + 'payloads': dict((p, resource_ref) for p in patterns) + }, { + **common_definition_fields, + 'type': 'resource', + 'revision': 1, + 'dependencies': [], + 'scripts': [{'file': 'script.js'}] + }] + } + + with tempfile.TemporaryDirectory() as tmpdir_str: + tmpdir = Path(tmpdir_str) + + source_dir = tmpdir / 'src' + source_dir.mkdir() + + malcontent_dir = tmpdir / 'malcontent' + malcontent_dir.mkdir() + + license_text = get_as_str('license_text') + if license_text: + package_index_json['copyright'] = [{'file': 'COPYING'}] + (source_dir / 'COPYING').write_text(license_text) + + (source_dir / 'script.js').write_text(get_as_str('script_text')) + + (source_dir / 'index.json').write_text(json.dumps(package_index_json)) + + try: + builder_args = ['-s', str(source_dir), '-d', str(malcontent_dir)] + build.perform(builder_args, standalone_mode=False) + build.perform(['-s', str(source_dir), '-d', '/tmp/haketilodebug'], standalone_mode=False) + _app.get_haketilo_state().import_items(malcontent_dir) + except: + import traceback + traceback.print_exc() + return items_import({'invalid_ad_hoc_package': True}) + + return flask.redirect(flask.url_for('items.packages')) + +@bp.route('/import', methods=['POST']) +def items_import_post() -> werkzeug.Response: + action = flask.request.form['action'] + + if action == 'import_from_file': + return items_import_from_file() + elif action == 'import_ad_hoc': + return item_import_ad_hoc() + else: + raise ValueError() diff --git a/src/hydrilla/proxy/web_ui/root.py b/src/hydrilla/proxy/web_ui/root.py index 24ff73f..402c5ae 100644 --- a/src/hydrilla/proxy/web_ui/root.py +++ b/src/hydrilla/proxy/web_ui/root.py @@ -48,6 +48,7 @@ from .. import http_messages from . import rules from . import repos from . import items +from . import items_import from . import prompts from . import _app @@ -99,7 +100,9 @@ class WebUIAppImpl(_app.WebUIApp): self.before_request(authenticate_by_referrer) - for blueprint in [rules.bp, repos.bp, items.bp, prompts.bp]: + for blueprint in [ + rules.bp, repos.bp, items.bp, items_import.bp, prompts.bp + ]: self.register_blueprint(blueprint) # Flask app is not thread-safe and has to be accompanied by an ugly lock. This diff --git a/src/hydrilla/proxy/web_ui/templates/base.html.jinja b/src/hydrilla/proxy/web_ui/templates/base.html.jinja index c0eecce..eb9cb57 100644 --- a/src/hydrilla/proxy/web_ui/templates/base.html.jinja +++ b/src/hydrilla/proxy/web_ui/templates/base.html.jinja @@ -62,6 +62,36 @@ code in a proprietary work, I am not going to enforce this in court. {% endmacro %} +{% macro form_label(label_text, label_name=none) %} + {% set for_attr = label_name and (label_name + "_field") %} + +{% endmacro %} + +{% macro form_field(field_name, required=true, sep_after=true, height=none) %} +
+ {% + set attrs = { + 'id': field_name + '_field', + 'name': field_name, + 'required': '' if required else none, + 'rows': height + } + %} + + {% if height is none %} + + {% else %} + + {% endif %} +
+ + {% if sep_after %} +
+ {% endif %} +{% endmacro %} + {% block head %} @@ -134,6 +164,10 @@ code in a proprietary work, I am not going to enforce this in court. margin: 0 0 10px 0; } + textarea { + resize: none; + } + .green-button, .red-button, .blue-button { border: none; border-radius: 2px; @@ -243,7 +277,7 @@ code in a proprietary work, I am not going to enforce this in court. ('items.packages', _('web_ui.base.nav.packages'), false), ('items.libraries', _('web_ui.base.nav.libraries'), true), ('repos.repos', _('web_ui.base.nav.repos'), false), - ('items.load_from_disk', _('web_ui.base.nav.load'), false) + ('import.items_import', _('web_ui.base.nav.import'), false) ] %}