[proxy] make it possible to create "ad hoc" packages through the web UI

7 files changed, 360 insertions, 137 deletions

diff --git a/src/hydrilla/proxy/web_ui/items.py b/src/hydrilla/proxy/web_ui/items.py
index 6195107..f34b89b 100644
--- a/src/hydrilla/proxy/web_ui/items.py
+++ b/src/hydrilla/proxy/web_ui/items.py
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# Proxy web UI packages loading.
+# Proxy web UI package/library management.
 #
 # This file is part of Hydrilla&Haketilo.
 #
@@ -32,17 +32,11 @@
 # Enable using with Python 3.7.
 from __future__ import annotations
 
-import tempfile
-import zipfile
 import typing as t
 
-from pathlib import Path
-
 import flask
 import werkzeug
 
-from ...exceptions import HaketiloException
-from ...translations import smart_gettext as _
 from ... import item_infos
 from .. import state as st
 from . import _app
@@ -50,45 +44,6 @@ from . import _app
 
 bp = flask.Blueprint('items', __package__)
 
-@bp.route('/load_from_disk', methods=['GET'])
-def load_from_disk(errors: t.Mapping[str, bool] = {}) -> werkzeug.Response:
-    html = flask.render_template('items/load_from_disk.html.jinja', **errors)
-    return flask.make_response(html, 200)
-
-@bp.route('/load_from_disk', methods=['POST'])
-def load_from_disk_post() -> werkzeug.Response:
-    zip_file_storage = flask.request.files.get('items_zipfile')
-    if zip_file_storage is None:
-        return load_from_disk()
-
-    with tempfile.TemporaryDirectory() as tmpdir_str:
-        tmpdir = Path(tmpdir_str)
-        tmpdir_child = tmpdir / 'childdir'
-        tmpdir_child.mkdir()
-
-        try:
-            with zipfile.ZipFile(zip_file_storage) as zip_file:
-                zip_file.extractall(tmpdir_child)
-        except:
-            return load_from_disk({'uploaded_file_not_zip': True})
-
-        extracted_top_level_files = tuple(tmpdir_child.iterdir())
-        if extracted_top_level_files == ():
-            return load_from_disk({'invalid_uploaded_malcontent': True})
-
-        if len(extracted_top_level_files) == 1 and \
-           extracted_top_level_files[0].is_dir():
-            malcontent_dir_path = extracted_top_level_files[0]
-        else:
-            malcontent_dir_path = tmpdir_child
-
-        try:
-            _app.get_haketilo_state().import_items(malcontent_dir_path)
-        except:
-            return load_from_disk({'invalid_uploaded_malcontent': True})
-
-    return flask.redirect(flask.url_for('.packages'))
-
 @bp.route('/packages')
 def packages() -> werkzeug.Response:
     store = _app.get_haketilo_state().mapping_store()
diff --git a/src/hydrilla/proxy/web_ui/items_import.py b/src/hydrilla/proxy/web_ui/items_import.py
new file mode 100644
index 0000000..b335237
--- /dev/null
+++ b/src/hydrilla/proxy/web_ui/items_import.py
@@ -0,0 +1,196 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# Proxy web UI packages loading.
+#
+# This file is part of Hydrilla&Haketilo.
+#
+# Copyright (C) 2022 Wojtek Kosior
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+#
+# I, Wojtek Kosior, thereby promise not to sue for violation of this
+# file's license. Although I request that you do not make use of this
+# code in a proprietary program, I am not going to enforce this in
+# court.
+
+"""
+.....
+"""
+
+# Enable using with Python 3.7.
+from __future__ import annotations
+
+import tempfile
+import zipfile
+import re
+import json
+import typing as t
+
+from pathlib import Path
+
+import flask
+import werkzeug
+
+from ...url_patterns import normalize_pattern
+from ...builder import build
+from ... import versions
+from .. import state as st
+from . import _app
+
+
+bp = flask.Blueprint('import', __package__)
+
+@bp.route('/import', methods=['GET'])
+def items_import(errors: t.Mapping[str, bool] = {}) -> werkzeug.Response:
+    html = flask.render_template('import.html.jinja', **errors)
+    return flask.make_response(html, 200)
+
+def items_import_from_file() -> werkzeug.Response:
+    zip_file_storage = flask.request.files.get('items_zipfile')
+    if zip_file_storage is None:
+        return items_import()
+
+    with tempfile.TemporaryDirectory() as tmpdir_str:
+        tmpdir = Path(tmpdir_str)
+        tmpdir_child = tmpdir / 'childdir'
+        tmpdir_child.mkdir()
+
+        try:
+            with zipfile.ZipFile(zip_file_storage) as zip_file:
+                zip_file.extractall(tmpdir_child)
+        except:
+            return items_import({'uploaded_file_not_zip': True})
+
+        extracted_top_level_files = tuple(tmpdir_child.iterdir())
+        if extracted_top_level_files == ():
+            return items_import({'invalid_uploaded_malcontent': True})
+
+        if len(extracted_top_level_files) == 1 and \
+           extracted_top_level_files[0].is_dir():
+            malcontent_dir_path = extracted_top_level_files[0]
+        else:
+            malcontent_dir_path = tmpdir_child
+
+        try:
+            _app.get_haketilo_state().import_items(malcontent_dir_path)
+        except:
+            return items_import({'invalid_uploaded_malcontent': True})
+
+    return flask.redirect(flask.url_for('items.packages'))
+
+identifier_re = re.compile(r'^[-0-9a-z.]+$')
+
+def item_import_ad_hoc() -> werkzeug.Response:
+    form = flask.request.form
+    def get_as_str(field_name: str) -> str:
+        value = form[field_name]
+        assert isinstance(value, str)
+        return value.strip()
+
+    try:
+        identifier = get_as_str('identifier')
+        assert identifier
+        assert identifier_re.match(identifier)
+    except:
+        return items_import({'invalid_ad_hoc_identifier': True})
+
+    long_name = get_as_str('long_name') or identifier
+
+    resource_ref = {'identifier': identifier}
+
+    try:
+        ver = versions.parse(get_as_str('version') or '1')
+    except:
+        return items_import({'invalid_ad_hoc_version': True})
+
+    try:
+        pat_str = get_as_str('patterns')
+        patterns = [
+            normalize_pattern(p.strip())
+            for p in pat_str.split('\n')
+            if p and not p.isspace()
+        ]
+        assert patterns
+    except:
+        return items_import({'invalid_ad_hoc_patterns': True})
+
+    common_definition_fields: t.Mapping[str, t.Any] = {
+        'identifier':   identifier,
+        'long_name':    long_name,
+        'version':      ver,
+        'description':  get_as_str('description')
+    }
+
+    schema_url = \
+        'https://hydrilla.koszko.org/schemas/package_source-1.schema.json'
+
+    package_index_json = {
+        '$schema':      schema_url,
+        'source_name':  'haketilo-ad-hoc-package',
+        'copyright':    [],
+        'upstream_url': '<local ad hoc package>',
+        'definitions': [{
+            **common_definition_fields,
+            'type':     'mapping',
+            'payloads': dict((p, resource_ref) for p in patterns)
+        }, {
+            **common_definition_fields,
+            'type':         'resource',
+            'revision':     1,
+            'dependencies': [],
+            'scripts':      [{'file': 'script.js'}]
+        }]
+    }
+
+    with tempfile.TemporaryDirectory() as tmpdir_str:
+        tmpdir = Path(tmpdir_str)
+
+        source_dir = tmpdir / 'src'
+        source_dir.mkdir()
+
+        malcontent_dir = tmpdir / 'malcontent'
+        malcontent_dir.mkdir()
+
+        license_text = get_as_str('license_text')
+        if license_text:
+            package_index_json['copyright'] = [{'file': 'COPYING'}]
+            (source_dir / 'COPYING').write_text(license_text)
+
+        (source_dir / 'script.js').write_text(get_as_str('script_text'))
+
+        (source_dir / 'index.json').write_text(json.dumps(package_index_json))
+
+        try:
+            builder_args = ['-s', str(source_dir), '-d', str(malcontent_dir)]
+            build.perform(builder_args, standalone_mode=False)
+            build.perform(['-s', str(source_dir), '-d', '/tmp/haketilodebug'], standalone_mode=False)
+            _app.get_haketilo_state().import_items(malcontent_dir)
+        except:
+            import traceback
+            traceback.print_exc()
+            return items_import({'invalid_ad_hoc_package': True})
+
+    return flask.redirect(flask.url_for('items.packages'))
+
+@bp.route('/import', methods=['POST'])
+def items_import_post() -> werkzeug.Response:
+    action = flask.request.form['action']
+
+    if action == 'import_from_file':
+        return items_import_from_file()
+    elif action == 'import_ad_hoc':
+        return item_import_ad_hoc()
+    else:
+        raise ValueError()
diff --git a/src/hydrilla/proxy/web_ui/root.py b/src/hydrilla/proxy/web_ui/root.py
index 24ff73f..402c5ae 100644
--- a/src/hydrilla/proxy/web_ui/root.py
+++ b/src/hydrilla/proxy/web_ui/root.py
@@ -48,6 +48,7 @@ from .. import http_messages
 from . import rules
 from . import repos
 from . import items
+from . import items_import
 from . import prompts
 from . import _app
 
@@ -99,7 +100,9 @@ class WebUIAppImpl(_app.WebUIApp):
 
         self.before_request(authenticate_by_referrer)
 
-        for blueprint in [rules.bp, repos.bp, items.bp, prompts.bp]:
+        for blueprint in [
+                rules.bp, repos.bp, items.bp, items_import.bp, prompts.bp
+        ]:
             self.register_blueprint(blueprint)
 
 # Flask app is not thread-safe and has to be accompanied by an ugly lock. This
diff --git a/src/hydrilla/proxy/web_ui/templates/base.html.jinja b/src/hydrilla/proxy/web_ui/templates/base.html.jinja
index c0eecce..eb9cb57 100644
--- a/src/hydrilla/proxy/web_ui/templates/base.html.jinja
+++ b/src/hydrilla/proxy/web_ui/templates/base.html.jinja
@@ -62,6 +62,36 @@ code in a proprietary work, I am not going to enforce this in court.
   </aside>
 {% endmacro %}
 
+{% macro form_label(label_text, label_name=none) %}
+  {% set for_attr = label_name and (label_name + "_field") %}
+  <label {{ {'for': for_attr, 'class': 'block-with-bottom-margin'}|xmlattr }}>
+    {{ label_text }}
+  </label>
+{% endmacro %}
+
+{% macro form_field(field_name, required=true, sep_after=true, height=none) %}
+  <div class="flex-row">
+    {%
+      set attrs = {
+          'id':       field_name + '_field',
+          'name':     field_name,
+          'required': '' if required else none,
+          'rows':     height
+      }
+    %}
+
+    {% if height is none %}
+      <input{{ attrs|xmlattr }}>
+    {% else %}
+      <textarea{{ attrs|xmlattr }}></textarea>
+    {% endif %}
+  </div>
+
+  {% if sep_after %}
+    <div class="horizontal-separator"></div>
+  {% endif %}
+{% endmacro %}
+
 <html>
   <head>
     {% block head %}
@@ -134,6 +164,10 @@ code in a proprietary work, I am not going to enforce this in court.
               margin: 0 0 10px 0;
           }
 
+          textarea {
+              resize: none;
+          }
+
           .green-button, .red-button, .blue-button {
                   border: none;
                   border-radius: 2px;
@@ -243,7 +277,7 @@ code in a proprietary work, I am not going to enforce this in court.
           ('items.packages',       _('web_ui.base.nav.packages'),  false),
           ('items.libraries',      _('web_ui.base.nav.libraries'), true),
           ('repos.repos',          _('web_ui.base.nav.repos'),     false),
-          ('items.load_from_disk', _('web_ui.base.nav.load'),      false)
+          ('import.items_import',  _('web_ui.base.nav.import'),    false)
       ]
     %}
     <ul id="nav">
diff --git a/src/hydrilla/proxy/web_ui/templates/import.html.jinja b/src/hydrilla/proxy/web_ui/templates/import.html.jinja
new file mode 100644
index 0000000..a8f16a4
--- /dev/null
+++ b/src/hydrilla/proxy/web_ui/templates/import.html.jinja
@@ -0,0 +1,120 @@
+{#
+SPDX-License-Identifier: GPL-3.0-or-later OR CC-BY-SA-4.0
+
+Proxy web UI item loading page.
+
+This file is part of Hydrilla&Haketilo.
+
+Copyright (C) 2022 Wojtek Kosior
+
+Dual licensed under
+* GNU General Public License v3.0 or later and
+* Creative Commons Attribution Share Alike 4.0 International.
+
+You can choose to use either of these licenses or both.
+
+
+I, Wojtek Kosior, thereby promise not to sue for violation of this
+file's licenses. Although I request that you do not make use of this
+code in a proprietary work, I am not going to enforce this in court.
+#}
+{% extends "base.html.jinja" %}
+
+{% block title %} {{ _('web_ui.import.title') }} {% endblock %}
+
+{% block style %}
+  {{ super() }}
+
+  input[type="file"]::-webkit-file-selector-button,
+  input[type="file"]::file-selector-button {
+      display: none;
+  }
+
+  input[type="file"] {
+      display: block;
+      font-size: inherit;
+      font-style: inherit;
+  }
+{% endblock %}
+
+{% block main %}
+  <h3>{{ _('web_ui.import.heading') }}</h3>
+
+  <h4>{{ _('web_ui.import.heading_import_from_file') }}</h4>
+
+  <form method="POST" enctype="multipart/form-data">
+    <input name="action" type="hidden" value="import_from_file">
+
+    {% if uploaded_file_not_zip is defined %}
+      {{ error_note(_('web_ui.err.uploaded_file_not_zip')) }}
+    {% endif %}
+
+    {% if invalid_uploaded_malcontent is defined %}
+      {{ error_note(_('web_ui.err.invalid_uploaded_malcontent')) }}
+    {% endif %}
+
+    <input id="items_zipfile" name="items_zipfile" type="file"
+           accept=".zip,application/zip" required=""
+           class="block-with-bottom-margin">
+
+    <label class="green-button block-with-bottom-margin" for="items_zipfile">
+      {{ _('web_ui.import.choose_zipfile_button') }}
+    </label>
+
+    <div class="horizontal-separator"></div>
+
+    <div class="flex-row">
+      <button class="green-button">
+        {{ _('web_ui.import.install_from_file_button') }}
+      </button>
+    </div>
+  </form>
+
+  <div class="horizontal-separator"></div>
+
+  <h4>{{ _('web_ui.import.heading_import_ad_hoc') }}</h4>
+
+  {% if invalid_ad_hoc_package is defined %}
+    {{ error_note(_('web_ui.err.invalid_ad_hoc_package')) }}
+  {% endif %}
+
+  <form method="POST">
+    <input name="action" type="hidden" value="import_ad_hoc">
+
+    {{ form_label(_('web_ui.import.identifier_field_label'), 'identifier') }}
+    {% if invalid_ad_hoc_identifier is defined %}
+      {{ error_note(_('web_ui.err.invalid_ad_hoc_identifier')) }}
+    {% endif %}
+    {{ form_field('identifier') }}
+
+    {{ form_label(_('web_ui.import.long_name_field_label'), 'long_name') }}
+    {{ form_field('long_name', required=false) }}
+
+    {{ form_label(_('web_ui.import.version_field_label'), 'version') }}
+    {% if invalid_ad_hoc_version is defined %}
+      {{ error_note(_('web_ui.err.invalid_ad_hoc_version')) }}
+    {% endif %}
+    {{ form_field('version', required=false) }}
+
+    {{ form_label(_('web_ui.import.description_field_label'), 'description') }}
+    {{ form_field('description', required=false, height=3) }}
+
+    {{ form_label(_('web_ui.import.patterns_field_label'), 'patterns') }}
+    {% if invalid_ad_hoc_patterns is defined %}
+      {{ error_note(_('web_ui.err.invalid_ad_hoc_patterns')) }}
+    {% endif %}
+    {{ form_field('patterns', height=3) }}
+
+    {{ form_label(_('web_ui.import.script_text_field_label'), 'script_text') }}
+    {{ form_field('script_text', required=false, height=15) }}
+
+    {{ form_label(_('web_ui.import.lic_text_field_label'), 'license_text') }}
+    {{ form_field('license_text', required=false, height=10) }}
+
+    <div class="flex-row">
+      <button class="green-button">
+        {{ _('web_ui.import.install_ad_hoc_button') }}
+      </button>
+    </div>
+  </form>
+{% endblock %}
diff --git a/src/hydrilla/proxy/web_ui/templates/items/load_from_disk.html.jinja b/src/hydrilla/proxy/web_ui/templates/items/load_from_disk.html.jinja
deleted file mode 100644
index f5d7b34..0000000
--- a/src/hydrilla/proxy/web_ui/templates/items/load_from_disk.html.jinja
+++ /dev/null
@@ -1,68 +0,0 @@
-{#
-SPDX-License-Identifier: GPL-3.0-or-later OR CC-BY-SA-4.0
-
-Proxy web UI item loading page.
-
-This file is part of Hydrilla&Haketilo.
-
-Copyright (C) 2022 Wojtek Kosior
-
-Dual licensed under
-* GNU General Public License v3.0 or later and
-* Creative Commons Attribution Share Alike 4.0 International.
-
-You can choose to use either of these licenses or both.
-
-
-I, Wojtek Kosior, thereby promise not to sue for violation of this
-file's licenses. Although I request that you do not make use of this
-code in a proprietary work, I am not going to enforce this in court.
-#}
-{% extends "base.html.jinja" %}
-
-{% block title %} {{ _('web_ui.load_from_disk.title') }} {% endblock %}
-
-{% block style %}
-  {{ super() }}
-
-  input[type="file"]::-webkit-file-selector-button,
-  input[type="file"]::file-selector-button {
-      display: none;
-  }
-
-  input[type="file"] {
-      display: block;
-      font-size: inherit;
-      font-style: inherit;
-  }
-{% endblock %}
-
-{% block main %}
-  <h3>{{ _('web_ui.load_from_disk.heading') }}</h3>
-
-  <form method="POST" enctype="multipart/form-data">
-    {% if uploaded_file_not_zip is defined %}
-      {{ error_note(_('web_ui.err.uploaded_file_not_zip')) }}
-    {% endif %}
-
-    {% if invalid_uploaded_malcontent is defined %}
-      {{ error_note(_('web_ui.err.invalid_uploaded_malcontent')) }}
-    {% endif %}
-
-    <input id="items_zipfile" name="items_zipfile" type="file"
-           accept=".zip,application/zip" required=""
-           class="block-with-bottom-margin">
-
-    <label class="green-button block-with-bottom-margin" for="items_zipfile">
-      {{ _('web_ui.load_from_disk.choose_zipfile_button') }}
-    </label>
-
-    <div class="horizontal-separator"></div>
-
-    <div class="flex-row">
-      <button class="green-button">
-        {{ _('web_ui.load_from_disk.install_button') }}
-      </button>
-    </div>
-  </form>
-{% endblock %}
diff --git a/src/hydrilla/proxy/web_ui/templates/repos/add.html.jinja b/src/hydrilla/proxy/web_ui/templates/repos/add.html.jinja
index 6bc1c19..be4933a 100644
--- a/src/hydrilla/proxy/web_ui/templates/repos/add.html.jinja
+++ b/src/hydrilla/proxy/web_ui/templates/repos/add.html.jinja
@@ -25,37 +25,20 @@ code in a proprietary work, I am not going to enforce this in court.
 {% block main %}
   <h3>{{ _('web_ui.repos.add.heading') }}</h3>
   <form method="POST">
+    {{ form_label(_('web_ui.repos.add.name_field_label'), 'name') }}
     {% if repo_name_invalid is defined %}
       {{ error_note(_('web_ui.err.repo_name_invalid')) }}
     {% endif %}
-
     {% if repo_name_taken is defined %}
       {{ error_note(_('web_ui.err.repo_name_taken')) }}
     {% endif %}
+    {{ form_field('name') }}
 
-    <label for="name_field" class="block-with-bottom-margin">
-      {{ _('web_ui.repos.add.name_field_label') }}
-    </label>
-
-    <div class="flex-row">
-      <input id="name_field" name="name" required="">
-    </div>
-
-    <div class="horizontal-separator"></div>
-
+    {{ form_label(_('web_ui.repos.add.url_field_label'), 'url') }}
     {% if repo_url_invalid is defined %}
       {{ error_note(_('web_ui.err.repo_url_invalid')) }}
     {% endif %}
-
-    <label for="url_field" class="block-with-bottom-margin">
-      {{ _('web_ui.repos.add.url_field_label') }}
-    </label>
-
-    <div class="flex-row">
-      <input id="url_field" name="url" required="">
-    </div>
-
-    <div class="horizontal-separator"></div>
+    {{ form_field('url') }}
 
     <div class="flex-row block-with-bottom-margin">
       <button class="green-button">