diff options
| -rw-r--r-- | PKG-INFO | 28 | ||||
| -rw-r--r-- | README.md | 26 | ||||
| -rw-r--r-- | conftest.py | 93 | ||||
| -rw-r--r-- | doc/examples/hydrilla.wsgi | 3 | ||||
| -rw-r--r-- | doc/man/man1/hydrilla.1 | 19 | ||||
| -rw-r--r-- | setup.cfg | 8 | ||||
| -rwxr-xr-x | setup.py | 37 | ||||
| -rw-r--r-- | src/hydrilla.egg-info/PKG-INFO | 28 | ||||
| -rw-r--r-- | src/hydrilla.egg-info/SOURCES.txt | 2 | ||||
| -rw-r--r-- | src/hydrilla.egg-info/requires.txt | 5 | ||||
| -rw-r--r-- | src/hydrilla/server/_version.py | 4 | ||||
| -rw-r--r-- | src/hydrilla/server/locales/en_US/LC_MESSAGES/hydrilla-messages.po | 66 | ||||
| -rw-r--r-- | src/hydrilla/server/serve.py | 37 | ||||
| -rw-r--r-- | tests/helpers.py | 51 | ||||
| -rw-r--r-- | tests/source-package-example/index.json | 154 | ||||
| -rw-r--r-- | tests/test_server.py | 154 |
16 files changed, 503 insertions, 212 deletions
@@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: hydrilla -Version: 1.0 +Version: 1.1b1 Summary: Hydrilla repository server Home-page: https://git.koszko.org/pydrilla Author: Wojtek Kosior @@ -19,21 +19,21 @@ Description: # Hydrilla (Python implementation) * Python3 (>= 3.7) * [hydrilla.builder](https://git.koszko.org/hydrilla-builder/) - * flask + * flask (>= 1.1) * click * jsonschema (>= 3.0) ### Build + * build (a PEP517 package builder) * setuptools * wheel * setuptools_scm - * babel + * babel (Python library) ### Test * pytest - * reuse ## Building @@ -55,15 +55,11 @@ Description: # Hydrilla (Python implementation) To perform the build and installation without PyPI, first install all dependencies system-wide. For example, in Debian-based distributions (including Trisquel): ``` shell sudo apt install python3-flask python3-flask python3-jsonschema \ - python3-setuptools python3-setuptools-scm python3-babel python3-wheel + python3-setuptools python3-setuptools-scm python3-babel python3-wheel \ + python3-build ``` - Then, block programs you're about to spawn from accessing https://pypi.org. If running on a GNU/Linux system you can utilize Linux user namespaces: - ``` shell - unshare -Urn - ``` - - The above will put you in a network-isolated shell. If you're using a virtualenv, activate it **after** the `unshare` command. + If you're using `virtualenv` command to create a virtual environment, make sure you invoke it with `--system-site-packages` and `--no-download`. The first option is necessary for packages installed inside the virtualenv to be able to use globally-installed dependencies. The second one will make `virtualenv` use locally-available base libraries (setuptools, etc.) instead of downloading them from PyPI. Now, in unpacked source directories of **both** `hydrilla-builder` and `hydrilla`, run the build and installation commands: ``` shell @@ -81,6 +77,16 @@ Description: # Hydrilla (Python implementation) python3 -m pytest ``` + ## Installation from wheels + + Instead of building yourself you can use Python wheels provided on [Hydrilla downloads page](https://hydrillabugs.koszko.org/projects/hydrilla/wiki/Releases). + + ``` shell + python3 -m pip install \ + path/to/downloaded/hydrilla.builder-1.1b1-py3-none-any.whl \ + path/to/downloaded/hydrilla-1.1b1-py3-none-any.whl + ``` + ## Running ### Hydrilla command @@ -10,21 +10,21 @@ The information below is meant to help hack on the codebase. If you're instead l * Python3 (>= 3.7) * [hydrilla.builder](https://git.koszko.org/hydrilla-builder/) -* flask +* flask (>= 1.1) * click * jsonschema (>= 3.0) ### Build +* build (a PEP517 package builder) * setuptools * wheel * setuptools_scm -* babel +* babel (Python library) ### Test * pytest -* reuse ## Building @@ -46,15 +46,11 @@ Commands like `python3 -m build` and `python3 -m pip` but also `virtualenv` will To perform the build and installation without PyPI, first install all dependencies system-wide. For example, in Debian-based distributions (including Trisquel): ``` shell sudo apt install python3-flask python3-flask python3-jsonschema \ - python3-setuptools python3-setuptools-scm python3-babel python3-wheel + python3-setuptools python3-setuptools-scm python3-babel python3-wheel \ + python3-build ``` -Then, block programs you're about to spawn from accessing https://pypi.org. If running on a GNU/Linux system you can utilize Linux user namespaces: -``` shell -unshare -Urn -``` - -The above will put you in a network-isolated shell. If you're using a virtualenv, activate it **after** the `unshare` command. +If you're using `virtualenv` command to create a virtual environment, make sure you invoke it with `--system-site-packages` and `--no-download`. The first option is necessary for packages installed inside the virtualenv to be able to use globally-installed dependencies. The second one will make `virtualenv` use locally-available base libraries (setuptools, etc.) instead of downloading them from PyPI. Now, in unpacked source directories of **both** `hydrilla-builder` and `hydrilla`, run the build and installation commands: ``` shell @@ -72,6 +68,16 @@ For tests to pass you need compiled message catalogs to be present. If you've pe python3 -m pytest ``` +## Installation from wheels + +Instead of building yourself you can use Python wheels provided on [Hydrilla downloads page](https://hydrillabugs.koszko.org/projects/hydrilla/wiki/Releases). + +``` shell +python3 -m pip install \ + path/to/downloaded/hydrilla.builder-1.1b1-py3-none-any.whl \ + path/to/downloaded/hydrilla-1.1b1-py3-none-any.whl +``` + ## Running ### Hydrilla command diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..37401e8 --- /dev/null +++ b/conftest.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: CC0-1.0 + +# Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> +# +# Available under the terms of Creative Commons Zero v1.0 Universal. + +import sys +from pathlib import Path + +import pytest +import pkgutil +import importlib +import functools +from tempfile import TemporaryDirectory +from typing import Iterable + +here = Path(__file__).resolve().parent +sys.path.insert(0, str(here / 'src')) + +@pytest.fixture(autouse=True) +def no_requests(monkeypatch): + """Remove requests.sessions.Session.request for all tests.""" + if importlib.util.find_spec("requests") is not None: + monkeypatch.delattr('requests.sessions.Session.request') + +def _mock_subprocess_run(monkeypatch, where, mocked_run): + """Temporarily replace subprocess.run() with the given function.""" + class MockedSubprocess: + """Minimal mocked version of the subprocess module.""" + run = mocked_run + + monkeypatch.setattr(where, 'subprocess', MockedSubprocess) + +@pytest.fixture +def mock_subprocess_run(monkeypatch, request): + """ + Facilitate temporarily replacing subprocess.run() with a different function. + + If the 'subprocess_run' pytest marker has been used, perform the replacement + for the module-function pair supplied through it. + + Return a function that can be called to perform the same replacement in + another fixture or from inside a test function. + """ + mocker = functools.partial(_mock_subprocess_run, monkeypatch) + + marker = request.node.get_closest_marker('subprocess_run') + if marker: + where, mocked_run = marker.args + mocker(where, mocked_run) + + return mocker + +@pytest.fixture(autouse=True) +def no_gettext(monkeypatch, request): + """ + Make gettext return all strings untranslated unless we request otherwise. + """ + if request.node.get_closest_marker('enable_gettext'): + return + + import hydrilla + modules_to_process = [hydrilla] + + def add_child_modules(parent): + """ + Recursuvely collect all modules descending from 'parent' into an array. + """ + try: + load_paths = parent.__path__ + except AttributeError: + return + + for module_info in pkgutil.iter_modules(load_paths): + if module_info.name != '__main__': + __import__(f'{parent.__name__}.{module_info.name}') + modules_to_process.append(getattr(parent, module_info.name)) + add_child_modules(getattr(parent, module_info.name)) + + add_child_modules(hydrilla) + + for module in modules_to_process: + if hasattr(module, '_'): + monkeypatch.setattr(module, '_', lambda message: message) + +@pytest.fixture +def tmpdir() -> Iterable[Path]: + """ + Provide test case with a temporary directory that will be automatically + deleted after the test. + """ + with TemporaryDirectory() as tmpdir: + yield Path(tmpdir) diff --git a/doc/examples/hydrilla.wsgi b/doc/examples/hydrilla.wsgi index 70dd895..480722d 100644 --- a/doc/examples/hydrilla.wsgi +++ b/doc/examples/hydrilla.wsgi @@ -23,5 +23,6 @@ application = start_wsgi(standalone_mode=False) # Comment the above and uncomment this to use a different config file. #from hydrilla.server import config +#from pathlib import Path #application = start_wsgi(standalone_mode=False, -# obj=config.load(['/path/to/config.json'])) +# obj=config.load([Path('/path/to/config.json')])) diff --git a/doc/man/man1/hydrilla.1 b/doc/man/man1/hydrilla.1 index c71428c..9fff2f3 100644 --- a/doc/man/man1/hydrilla.1 +++ b/doc/man/man1/hydrilla.1 @@ -6,7 +6,7 @@ .\" .\" Available under the terms of Creative Commons Zero v1.0 Universal. -.TH HYDRILLA 1 2022-04-22 "Hydrilla 1.0" "Hydrilla Manual" +.TH HYDRILLA 1 2022-06-14 "Hydrilla 1.1" "Hydrilla Manual" .SH NAME hydrilla \- Serve packages for the Haketilo browser extension @@ -47,7 +47,9 @@ pun on widespread use of word .UE with regard to works published online. -This option, if present, overrides the property \*(lqmalcontent_dir\*(rq from Hydrilla config file. If the value is not specified on the command line nor in the config file, it defaults to \%\*(lq/var/lib/hydrilla/malcontent\*(rq. +This option, if present, overrides the property \*(lqmalcontent_dir\*(rq from +Hydrilla config file. If the value is not specified on the command line nor in +the config file, it defaults to \%\*(lq/var/lib/hydrilla/malcontent\*(rq. .TP .BI \-h " URL" "\fR,\fP \-\^\-hydrilla\-project\-url=" URL @@ -55,7 +57,10 @@ Use .I URL when placing a link to Hydrilla website in served HTML pages. -This option, if present, overrides the property \*(lqhydrilla_project_url\*(rq from Hydrilla config file. If the value is not specified on the command line nor in the config file, it defaults to \%\*(lqhttps://hydrillabugs.koszko.org/projects/hydrilla/wiki\*(rq. +This option, if present, overrides the property \*(lqhydrilla_project_url\*(rq +from Hydrilla config file. If the value is not specified on the command line nor +in the config file, it defaults to +\%\*(lqhttps://hydrillabugs.koszko.org/projects/hydrilla/wiki\*(rq. .TP .BI \-p " PORT" "\fR,\fP \-\^\-port=" PORT @@ -67,7 +72,9 @@ is 0, let .I hydrilla choose a random free port on the machine. -This option, if present, overrides the property \*(lqport\*(rq from Hydrilla config file. If the value is not specified on the command line nor in the config file, it defaults to 10112. +This option, if present, overrides the property \*(lqport\*(rq from Hydrilla +config file. If the value is not specified on the command line nor in the config +file, it defaults to 10112. .TP .BI \-l " LANGUAGE" "\fR,\fP \-\^\-language=" LANGUAGE @@ -83,7 +90,9 @@ Otherwise, .I hydrilla will silently fall back to the en_US locale. -This option, if present, overrides the property \*(lqlanguage\*(rq from Hydrilla config file. If the value is not specified on the command line nor in the config file, it defaults to \*(lqen_US\*(rq. +This option, if present, overrides the property \*(lqlanguage\*(rq from Hydrilla +config file. If the value is not specified on the command line nor in the config +file, it defaults to \*(lqen_US\*(rq. .TP .BI \-c " CONFIG" "\fR,\fP \-\^\-config=" CONFIG @@ -25,12 +25,12 @@ classifiers = zip_safe = False package_dir = = src -packages = find: +packages = find_namespace: include_package_data = True python_requires = >= 3.7 install_requires = - hydrilla.builder==1.0 - flask + hydrilla.builder==1.1b1 + flask>=1.1 jsonschema>=3.0 [options.package_data] @@ -38,7 +38,7 @@ hydrilla.server = locales/*/LC_MESSAGES/hydrilla-messages.mo [options.extras_require] test = pytest -setup = setuptools_scm +setup = setuptools_scm; babel [options.packages.find] where = src @@ -8,13 +8,42 @@ import setuptools from setuptools.command.build_py import build_py +from setuptools.command.sdist import sdist + +from pathlib import Path + +here = Path(__file__).resolve().parent class CustomBuildCommand(build_py): - ''' - The build command but runs babel before build. - ''' + """The build command but runs babel before build.""" def run(self, *args, **kwargs): + """Wrapper around build_py's original run() method.""" self.run_command('compile_catalog') + + super().run(*args, **kwargs) + +class CustomSdistCommand(sdist): + """ + The sdist command but prevents compiled message catalogs from being included + in the archive. + """ + def run(self, *args, **kwargs): + """Wrapper around sdist's original run() method.""" + locales_dir = here / 'src/hydrilla/server/locales' + locale_files = {} + + for path in locales_dir.rglob('*.mo'): + locale_files[path] = path.read_bytes() + + for path in locale_files: + path.unlink() + super().run(*args, **kwargs) -setuptools.setup(cmdclass={'build_py': CustomBuildCommand}) + for path, contents in locale_files.items(): + path.write_bytes(contents) + +setuptools.setup(cmdclass = { + 'build_py': CustomBuildCommand, + 'sdist': CustomSdistCommand +}) diff --git a/src/hydrilla.egg-info/PKG-INFO b/src/hydrilla.egg-info/PKG-INFO index dc23bac..b7cad65 100644 --- a/src/hydrilla.egg-info/PKG-INFO +++ b/src/hydrilla.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: hydrilla -Version: 1.0 +Version: 1.1b1 Summary: Hydrilla repository server Home-page: https://git.koszko.org/pydrilla Author: Wojtek Kosior @@ -19,21 +19,21 @@ Description: # Hydrilla (Python implementation) * Python3 (>= 3.7) * [hydrilla.builder](https://git.koszko.org/hydrilla-builder/) - * flask + * flask (>= 1.1) * click * jsonschema (>= 3.0) ### Build + * build (a PEP517 package builder) * setuptools * wheel * setuptools_scm - * babel + * babel (Python library) ### Test * pytest - * reuse ## Building @@ -55,15 +55,11 @@ Description: # Hydrilla (Python implementation) To perform the build and installation without PyPI, first install all dependencies system-wide. For example, in Debian-based distributions (including Trisquel): ``` shell sudo apt install python3-flask python3-flask python3-jsonschema \ - python3-setuptools python3-setuptools-scm python3-babel python3-wheel + python3-setuptools python3-setuptools-scm python3-babel python3-wheel \ + python3-build ``` - Then, block programs you're about to spawn from accessing https://pypi.org. If running on a GNU/Linux system you can utilize Linux user namespaces: - ``` shell - unshare -Urn - ``` - - The above will put you in a network-isolated shell. If you're using a virtualenv, activate it **after** the `unshare` command. + If you're using `virtualenv` command to create a virtual environment, make sure you invoke it with `--system-site-packages` and `--no-download`. The first option is necessary for packages installed inside the virtualenv to be able to use globally-installed dependencies. The second one will make `virtualenv` use locally-available base libraries (setuptools, etc.) instead of downloading them from PyPI. Now, in unpacked source directories of **both** `hydrilla-builder` and `hydrilla`, run the build and installation commands: ``` shell @@ -81,6 +77,16 @@ Description: # Hydrilla (Python implementation) python3 -m pytest ``` + ## Installation from wheels + + Instead of building yourself you can use Python wheels provided on [Hydrilla downloads page](https://hydrillabugs.koszko.org/projects/hydrilla/wiki/Releases). + + ``` shell + python3 -m pip install \ + path/to/downloaded/hydrilla.builder-1.1b1-py3-none-any.whl \ + path/to/downloaded/hydrilla-1.1b1-py3-none-any.whl + ``` + ## Running ### Hydrilla command diff --git a/src/hydrilla.egg-info/SOURCES.txt b/src/hydrilla.egg-info/SOURCES.txt index 461f1fc..d30f96a 100644 --- a/src/hydrilla.egg-info/SOURCES.txt +++ b/src/hydrilla.egg-info/SOURCES.txt @@ -3,6 +3,7 @@ MANIFEST.in README.md README.md.license babel.cfg +conftest.py pyproject.toml pytest.ini setup.cfg @@ -35,6 +36,7 @@ src/hydrilla/server/templates/base.html src/hydrilla/server/templates/index.html tests/__init__.py tests/config.json +tests/helpers.py tests/test_server.py tests/source-package-example/README.txt tests/source-package-example/README.txt.license diff --git a/src/hydrilla.egg-info/requires.txt b/src/hydrilla.egg-info/requires.txt index 026f10d..62e05e0 100644 --- a/src/hydrilla.egg-info/requires.txt +++ b/src/hydrilla.egg-info/requires.txt @@ -1,8 +1,9 @@ -flask -hydrilla.builder==1.0 +flask>=1.1 +hydrilla.builder==1.1b1 jsonschema>=3.0 [setup] +babel setuptools_scm [test] diff --git a/src/hydrilla/server/_version.py b/src/hydrilla/server/_version.py index d953eef..2feb153 100644 --- a/src/hydrilla/server/_version.py +++ b/src/hydrilla/server/_version.py @@ -1,5 +1,5 @@ # coding: utf-8 # file generated by setuptools_scm # don't change, don't track in version control -version = '1.0' -version_tuple = (1, 0) +version = '1.1b1' +version_tuple = (1, '1b1') diff --git a/src/hydrilla/server/locales/en_US/LC_MESSAGES/hydrilla-messages.po b/src/hydrilla/server/locales/en_US/LC_MESSAGES/hydrilla-messages.po index 7ea930a..1998f89 100644 --- a/src/hydrilla/server/locales/en_US/LC_MESSAGES/hydrilla-messages.po +++ b/src/hydrilla/server/locales/en_US/LC_MESSAGES/hydrilla-messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: hydrilla.builder 0.1\n" "Report-Msgid-Bugs-To: koszko@koszko.org\n" -"POT-Creation-Date: 2022-04-22 17:09+0200\n" +"POT-Creation-Date: 2022-05-31 18:21+0200\n" "PO-Revision-Date: 2022-02-12 00:00+0000\n" "Last-Translator: Wojtek Kosior <koszko@koszko.org>\n" "Language: en_US\n" @@ -18,56 +18,60 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.8.0\n" -#: src/hydrilla/server/serve.py:122 +#: src/hydrilla/server/serve.py:127 #, python-brace-format msgid "uuid_mismatch_{identifier}" msgstr "Two different uuids were specified for item '{identifier}'." -#: src/hydrilla/server/serve.py:129 +#: src/hydrilla/server/serve.py:134 #, python-brace-format msgid "version_clash_{identifier}_{version}" msgstr "Version '{version}' specified more than once for item '{identifier}'." -#: src/hydrilla/server/serve.py:245 src/hydrilla/server/serve.py:257 +#: src/hydrilla/server/serve.py:250 src/hydrilla/server/serve.py:262 msgid "invalid_URL_{}" msgstr "Invalid URL/pattern: '{}'." -#: src/hydrilla/server/serve.py:249 +#: src/hydrilla/server/serve.py:254 msgid "disallowed_protocol_{}" msgstr "Disallowed protocol: '{}'." -#: src/hydrilla/server/serve.py:302 +#: src/hydrilla/server/serve.py:307 msgid "malcontent_dir_path_not_dir_{}" msgstr "Provided 'malcontent_dir' path does not name a directory: {}" -#: src/hydrilla/server/serve.py:321 +#: src/hydrilla/server/serve.py:326 msgid "couldnt_load_item_from_{}" msgstr "Couldn't load item from {}." -#: src/hydrilla/server/serve.py:347 +#: src/hydrilla/server/serve.py:351 msgid "item_{item}_in_file_{file}" msgstr "Item {item} incorrectly present under {file}." -#: src/hydrilla/server/serve.py:353 +#: src/hydrilla/server/serve.py:357 msgid "item_version_{ver}_in_file_{file}" msgstr "Item version {ver} incorrectly present under {file}." -#: src/hydrilla/server/serve.py:376 +#: src/hydrilla/server/serve.py:380 msgid "no_dep_{resource}_{ver}_{dep}" msgstr "Unknown dependency '{dep}' of resource '{resource}', version '{ver}'." -#: src/hydrilla/server/serve.py:387 +#: src/hydrilla/server/serve.py:391 msgid "no_payload_{mapping}_{ver}_{payload}" msgstr "Unknown payload '{payload}' of mapping '{mapping}', version '{ver}'." -#: src/hydrilla/server/serve.py:413 +#: src/hydrilla/server/serve.py:403 +msgid "no_mapping_{required_by}_{ver}_{required}" +msgstr "Unknown mapping '{required}' required by '{required_by}', version '{ver}'." + +#: src/hydrilla/server/serve.py:430 msgid "couldnt_register_{mapping}_{ver}_{pattern}" msgstr "" "Couldn't register mapping '{mapping}', version '{ver}' (pattern " "'{pattern}')." -#: src/hydrilla/server/serve.py:566 src/hydrilla/server/serve.py:588 -#: src/hydrilla/server/serve.py:626 +#: src/hydrilla/server/serve.py:583 src/hydrilla/server/serve.py:606 +#: src/hydrilla/server/serve.py:650 #, python-format msgid "%(prog)s_%(version)s_license" msgstr "" @@ -78,55 +82,55 @@ msgstr "" "This is free software: you are free to change and redistribute it.\n" "There is NO WARRANTY, to the extent permitted by law." -#: src/hydrilla/server/serve.py:577 +#: src/hydrilla/server/serve.py:592 +msgid "serve_hydrilla_packages_explain_wsgi_considerations" +msgstr "" +"Serve Hydrilla packages.\n" +"\n" +"This command is meant to be a quick way to run a local or development " +"Hydrilla instance. For better performance, consider deployment using " +"WSGI." + +#: src/hydrilla/server/serve.py:595 msgid "directory_to_serve_from_overrides_config" msgstr "" "Directory to serve files from. Overrides value from the config file (if " "any)." -#: src/hydrilla/server/serve.py:579 +#: src/hydrilla/server/serve.py:597 msgid "project_url_to_display_overrides_config" msgstr "" "Project url to display on generated HTML pages. Overrides value from the " "config file (if any)." -#: src/hydrilla/server/serve.py:581 +#: src/hydrilla/server/serve.py:599 msgid "tcp_port_to_listen_on_overrides_config" msgstr "" "TCP port number to listen on (0-65535). Overrides value from the config " "file (if any)." -#: src/hydrilla/server/serve.py:584 +#: src/hydrilla/server/serve.py:602 msgid "path_to_config_file_explain_default" msgstr "" "Path to Hydrilla server configuration file (optional, by default Hydrilla" " loads its own config file, which in turn tries to load " "/etc/hydrilla/config.json)." -#: src/hydrilla/server/serve.py:586 +#: src/hydrilla/server/serve.py:604 msgid "language_to_use_overrides_config" msgstr "" "Language to use (also affects served HTML files). Overrides value from " "the config file (if any)." -#: src/hydrilla/server/serve.py:589 src/hydrilla/server/serve.py:627 +#: src/hydrilla/server/serve.py:607 src/hydrilla/server/serve.py:651 msgid "version_printing" msgstr "Print version information and exit." -#: src/hydrilla/server/serve.py:617 +#: src/hydrilla/server/serve.py:640 msgid "config_option_{}_not_supplied" msgstr "Missing configuration option '{}'." -#: src/hydrilla/server/serve.py:621 -msgid "serve_hydrilla_packages_explain_wsgi_considerations" -msgstr "" -"Serve Hydrilla packages.\n" -"\n" -"This command is meant to be a quick way to run a local or development " -"Hydrilla instance. For better performance, consider deployment using " -"WSGI." - -#: src/hydrilla/server/serve.py:632 +#: src/hydrilla/server/serve.py:644 msgid "serve_hydrilla_packages_wsgi_help" msgstr "" "Serve Hydrilla packages.\n" diff --git a/src/hydrilla/server/serve.py b/src/hydrilla/server/serve.py index a6a1204..779f3d2 100644 --- a/src/hydrilla/server/serve.py +++ b/src/hydrilla/server/serve.py @@ -56,13 +56,18 @@ generated_by = { class ItemInfo(ABC): """Shortened data of a resource/mapping.""" - def __init__(self, item_obj: dict): + def __init__(self, item_obj: dict, major_schema_version: int): """Initialize ItemInfo using item definition read from JSON.""" self.version = util.normalize_version(item_obj['version']) self.identifier = item_obj['identifier'] self.uuid = item_obj.get('uuid') self.long_name = item_obj['long_name'] + self.required_mappings = [] + if major_schema_version >= 2: + self.required_mappings = [map_ref['identifier'] for map_ref in + item_obj.get('required_mappings', [])] + def path(self) -> str: """ Get a relative path to this item's JSON definition with respect to @@ -72,18 +77,18 @@ class ItemInfo(ABC): class ResourceInfo(ItemInfo): """Shortened data of a resource.""" - def __init__(self, resource_obj: dict): + def __init__(self, resource_obj: dict, major_schema_version: int): """Initialize ResourceInfo using resource definition read from JSON.""" - super().__init__(resource_obj) + super().__init__(resource_obj, major_schema_version) dependencies = resource_obj.get('dependencies', []) self.dependencies = [res_ref['identifier'] for res_ref in dependencies] class MappingInfo(ItemInfo): """Shortened data of a mapping.""" - def __init__(self, mapping_obj: dict): + def __init__(self, mapping_obj: dict, major_schema_version: int): """Initialize MappingInfo using mapping definition read from JSON.""" - super().__init__(mapping_obj) + super().__init__(mapping_obj, major_schema_version) self.payloads = {} for pattern, res_ref in mapping_obj.get('payloads', {}).items(): @@ -332,16 +337,15 @@ class Malcontent: version = util.parse_version(ver_file.name) identifier = ver_file.parent.name - with open(ver_file, 'rt') as file_handle: - item_json = json.load(file_handle) + item_json, major = util.load_instance_from_file(ver_file) - util.validator_for(f'api_{item_type}_description-1.0.1.schema.json')\ + util.validator_for(f'api_{item_type}_description-{major}.schema.json')\ .validate(item_json) if item_type == 'resource': - item_info = ResourceInfo(item_json) + item_info = ResourceInfo(item_json, major) else: - item_info = MappingInfo(item_json) + item_info = MappingInfo(item_json, major) if item_info.identifier != identifier: msg = f_('item_{item}_in_file_{file}')\ @@ -394,6 +398,19 @@ class Malcontent: if payload not in self.infos['resource']: report_missing_payload(mapping_info, payload) + def report_missing_mapping(info: Union[MappingInfo, ResourceInfo], + required_mapping: str) -> None: + msg = _('no_mapping_{required_by}_{ver}_{required}')\ + .format(required_by=info.identifier, required=required_mapping, + ver=util.version_string(info.version)) + logging.error(msg) + + for item_info in (*self._all_of_type('mapping'), + *self._all_of_type('resource')): + for required in item_info.required_mappings: + if required not in self.infos['mapping']: + report_missing_mapping(item_info, required) + def _finalize(self): """ Initialize structures needed to serve queries. Called once after all diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..df474b0 --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: CC0-1.0 + +# Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> +# +# Available under the terms of Creative Commons Zero v1.0 Universal. + +import re + +variable_word_re = re.compile(r'^<(.+)>$') + +def process_command(command, expected_command): + """Validate the command line and extract its variable parts (if any).""" + assert len(command) == len(expected_command) + + extracted = {} + for word, expected_word in zip(command, expected_command): + match = variable_word_re.match(expected_word) + if match: + extracted[match.group(1)] = word + else: + assert word == expected_word + + return extracted + +def run_missing_executable(command, **kwargs): + """ + Instead of running a command, raise FileNotFoundError as if its executable + was missing. + """ + raise FileNotFoundError('dummy') + +class MockedCompletedProcess: + """ + Object with some fields similar to those of subprocess.CompletedProcess. + """ + def __init__(self, args, returncode=0, + stdout='some output', stderr='some error output', + text_output=True): + """ + Initialize MockedCompletedProcess. Convert strings to bytes if needed. + """ + self.args = args + self.returncode = returncode + + if type(stdout) is str and not text_output: + stdout = stdout.encode() + if type(stderr) is str and not text_output: + stderr = stderr.encode() + + self.stdout = stdout + self.stderr = stderr diff --git a/tests/source-package-example/index.json b/tests/source-package-example/index.json index 7162dd7..9aa6e70 100644 --- a/tests/source-package-example/index.json +++ b/tests/source-package-example/index.json @@ -3,9 +3,9 @@ // Copyright (C) 2021, 2022 Wojtek Kosior <koszko@koszko.org> // Available under the terms of Creative Commons Zero v1.0 Universal. -// This is an example index.json file describing Hydrilla site content. As you -// can see, for storing site content information Hydrilla utilizes JSON with an -// additional extension in the form of '//' comments support. +// This is an example index.json file describing Hydrilla packages. As you can +// see, for storing this information Hydrilla utilizes JSON with an additional +// extension in the form of '//' comments support. // An index.json file conveys definitions of site resources and pattern->payload // mappings. The definitions may reference files under index.json's containing @@ -62,18 +62,15 @@ // will also belong here once they get implemented. "definitions": [ { - // Value of "type" can currently be one of: "resource" and - // "mapping". The one we have here, "resource", defines a list - // of injectable scripts that can be used as a payload or as a - // dependency of another "resource". In the future CSS style sheets - // and WASM modules will also be composite parts of a "resource" as - // scripts are now. - "type": "resource", + // Value of "type" can currently be one of: "resource", "mapping" + // and "mapping_and_resource" for a combined definition. The one we + // have here, "mapping", associates resources with pages on which + // they are to be used. + "type": "mapping", - // Used when referring to this resource in "dependencies" list of - // another resource or in "payload" field of a mapping. Should - // be consize and can only use a restricted set of characters. It - // has to match: [-0-9a-z]+ + // Used when referring to this mapping in "required_mappings" list + // of another item. Should be consize and can only use a restricted + // set of characters. It has to match: [-0-9a-z]+ "identifier": "helloapple", // "long_name" should be used to specify a user-friendly alternative @@ -96,26 +93,80 @@ // Different versions (e.g. 1.0 and 1.3) of the same resource can be // defined in separate index.json files. This makes it easy to // accidently cause an identifier clash. To help detect it, we allow - // each resource to have a UUID associated with it. Attempt to - // define multiple resources with the same identifier and different - // UUIDs will result in an error being reported. Defining multiple - // resources with different identifiers and the same UUID is + // each item to have a UUID associated with it. Attempt to define + // multiple mapping with the same identifier and different UUIDs + // will result in an error being reported. Defining multiple + // mappings with different identifiers and the same UUID is // disallowed for now (it may be later permitted if we consider it // good for some use-case). - // As of package source schema version 1.0, UUIDs are optional and + // As of package source schema version 2.0, UUIDs are optional and // can be omitted. + "uuid": "54d23bba-472e-42f5-9194-eaa24c0e3ee7", + + // Thanks to the "version" field (and "revision" field in case of + // "resource" or "mapping_and_resource", clients will know they have + // to update certain item after a new version has appeared in the + // repository. If multiple definitions of the same version of given + // item are provided to Hydrilla server, an error is generated. + // "version" differs from its counterpart in resource in that it has + // no accompanying revision number. For combined definitions with + // "mapping_and_resource" as type, the value of "revision" is + // appended as the last component of the resulting mapping's + // version. If type is simply "mapping", revision number is ignored. + "version": [2021, 11, 10], + + // A short, meaningful description of what the mapping does. + "description": "causes apple to get greeted on Hydrillabugs issue tracker", + + // If needed, a "comment" field can be added to provide some + // additional information. + // "comment": "this resource something something", + + // The "payloads" object specifies which payloads are to be applied + // to which URLs. + "payloads": { + // Each key should be a valid Haketilo URL pattern. + "https://hydrillabugs.koszko.org/***": { + // Should be the name of an existing resource. The resource + // may, but doesn't have to, be defined in the same + // index.json file. + "identifier": "helloapple" + }, + // More associations may follow. + "https://hachettebugs.koszko.org/***": { + "identifier": "helloapple" + } + } + }, { + // A "resource" item defines a list of injectable scripts that can + // be used as a payload or as a dependency of another "resource". In + // the future CSS style sheets and WASM modules will also be + // composite parts of a "resource" as scripts are now. + "type": "resource", + + // Has similar function to mapping's identifier. Used when + // referring to this resource in "dependencies" list of another + // resource or in "payload" field of a mapping. Should be consize + // and can only use a restricted set of characters. It has to match: + // [-0-9a-z]+ + // It can be the same as some mapping identifier (those are + // different entities and are treated separately). + "identifier": "helloapple", + + // "long name" and "uuid" have the same meaning as in the case of + // resources and "uuid" is also optional. UUIDs of a resource and a + // mapping can technically be the same but it is recommended to + // avoid even this kind of repetition. + "long_name": "Hello Apple", "uuid": "a6754dcb-58d8-4b7a-a245-24fd7ad4cd68", // Version should match the upstream version of the resource (e.g. a - // version of javascript library). Revision number starts as 1 for + // version of JavaScript library). Revision number starts as 1 for // each new resource version and gets incremented by 1 each time a - // modification to the packaging of this version is done. Hydrilla - // will allow multiple definitions of the same resource to load, as - // long as their versions differ. Thanks to the "version" and - // "revision" fields, clients will know they have to update certain - // resource after it has been updated. If multiple definitions of - // the same version of given resource are provided, an error is - // generated (even if those definitions differ by revision number). + // modification to the packaging of this version is done. + // If multiple definitions of the same version of given resource are + // provided to Hydrilla server, an error is generated (even if those + // definitions differ by revision number). "version": [2021, 11, 10], "revision": 1, @@ -123,16 +174,15 @@ // what it does. "description": "greets an apple", - // If needed, a "comment" field can be added to provide some - // additional information. - // "comment": "this resource something something", + // A comment, if necessary. + // "comment": "blah blah because bleh" // Resource's "dependencies" array shall contain names of other // resources that (in case of scripts at least) should get evaluated // on a page before this resource's own scripts. "dependencies": [{"identifier": "hello-message"}], - // Array of javascript files that belong to this resource. + // Array of JavaScript files that belong to this resource. "scripts": [ {"file": "hello.js"}, {"file": "bye.js"} @@ -148,48 +198,6 @@ // If "dependencies" is empty, it can also be omitted. // "dependencies": [], "scripts": [{"file": "message.js"}] - }, { - "type": "mapping", - - // Has similar function to resource's identifier. Should be consize - // and can only use a restricted set of characters. It has to match: - // [-0-9a-z]+ - // It can be the same as some resource identifier (those are - // different entities and are treated separately). - "identifier": "helloapple", - - // "long name" and "uuid" have the same meaning as in the case of - // resources and "uuid" is also optional. UUIDs of a resource and a - // mapping can technically be the same but it is recommended to - // avoid even this kind of repetition. - "long_name": "Hello Apple", - "uuid": "54d23bba-472e-42f5-9194-eaa24c0e3ee7", - - // "version" differs from its counterpart in resource in that it has - // no accompanying revision number. - "version": [2021, 11, 10], - - // A short, meaningful description of what the mapping does. - "description": "causes apple to get greeted on Hydrillabugs issue tracker", - - // A comment, if necessary. - // "comment": "blah blah because bleh" - - // The "payloads" object specifies which payloads are to be applied - // to which URLs. - "payloads": { - // Each key should be a valid Haketilo URL pattern. - "https://hydrillabugs.koszko.org/***": { - // Should be the name of an existing resource. The resource - // may, but doesn't have to, be defined in the same - // index.json file. - "identifier": "helloapple" - }, - // More associations may follow. - "https://hachettebugs.koszko.org/***": { - "identifier": "helloapple" - } - } } ], // We can also list additional files to include in the produced source diff --git a/tests/test_server.py b/tests/test_server.py index 0820d5c..65ae8ce 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -31,6 +31,7 @@ import pytest import sys import shutil import json +import functools as ft from pathlib import Path from hashlib import sha256 @@ -42,9 +43,10 @@ from markupsafe import escape from werkzeug import Response from hydrilla import util as hydrilla_util -from hydrilla.builder import Build -from hydrilla.server import config, _version -from hydrilla.server.serve import HydrillaApp +from hydrilla.builder import build +from hydrilla.server import config, _version, serve + +from .helpers import * here = Path(__file__).resolve().parent config_path = here / 'config.json' @@ -55,13 +57,41 @@ expected_generated_by = { 'version': _version.version } -SetupMod = Optional[Callable['Setup', None]] +SetupMod = Optional[Callable[['Setup'], None]] source_files = ( 'index.json', 'hello.js', 'bye.js', 'message.js', 'README.txt', 'README.txt.license', '.reuse/dep5', 'LICENSES/CC0-1.0.txt' ) +def run_reuse(command, **kwargs): + """ + Instead of running a 'reuse' command, check if 'mock_reuse_missing' file + exists under root directory. If yes, raise FileNotFoundError as if 'reuse' + command was missing. If not, check if 'README.txt.license' file exists + in the requested directory and return zero if it does. + """ + expected = ['reuse', '--root', '<root>', + 'lint' if 'lint' in command else 'spdx'] + + root_path = Path(process_command(command, expected)['root']) + + if (root_path / 'mock_reuse_missing').exists(): + raise FileNotFoundError('dummy') + + is_reuse_compliant = (root_path / 'README.txt.license').exists() + + return MockedCompletedProcess(command, 1 - is_reuse_compliant, + stdout=f'dummy {expected[-1]} output', + text_output=kwargs.get('text')) + +@pytest.fixture +def mock_reuse(mock_subprocess_run): + """ + Mock the REUSE command when executed through subprocess.run() from serve.py. + """ + mock_subprocess_run(build, run_reuse) + class Setup: """ Facilitate preparing test malcontent directory, Hydrilla config file and the @@ -95,8 +125,8 @@ class Setup: if self._modify_before_build: self._modify_before_build(self) - build = Build(self.source_dir, self.index_json) - build.write_package_files(self.malcontent_dir) + build_obj = build.Build(self.source_dir, self.index_json) + build_obj.write_package_files(self.malcontent_dir) if self._modify_after_build: self._modify_after_build(self) @@ -114,48 +144,64 @@ class Setup: Provide app client that serves the objects from built sample package. """ if self._client is None: - app = HydrillaApp(self.config(), flask_config={'TESTING': True}) + app = serve.HydrillaApp(self.config(), + flask_config={'TESTING': True}) self._client = app.test_client() return self._client -def remove_all_uuids(setup: Setup) -> None: - """Modify sample packages before build to contain no (optional) UUIDs""" - index_json = (setup.source_dir / 'index.json').read_text() - index_json = json.loads(hydrilla_util.strip_json_comments(index_json)) +def index_json_modification(modify_index_json): + """Decorator for function that modifies index.json before build.""" + def handle_index_json(setup): + """Modify index.json before build.""" + index_path = setup.source_dir / 'index.json' + index_json, _ = hydrilla_util.load_instance_from_file(index_path) + + index_json = modify_index_json(index_json) or index_json + + index_json = f''' + // SPDX-License-Identifier: CC0-1.0 + // Copyright (C) 2021, 2022 Wojtek Kosior + {json.dumps(index_json)} + ''' + + index_path.write_text(index_json) + return handle_index_json + +@index_json_modification +def remove_all_uuids(index_json): + """Modify sample packages to contain no (optional) UUIDs""" for definition in index_json['definitions']: del definition['uuid'] - index_json = ("// SPDX-License-Identifier: CC0-1.0\n" + - "// Copyright (C) 2021, 2022 Wojtek Kosior\n" + - json.dumps(index_json)) +@index_json_modification +def bump_schema_v2(index_json) -> None: + """Modify sample packages to use version 2 of Hydrilla JSON schemas.""" + for definition in index_json['definitions']: + definition['min_haketilo_version'] = [1, 1] - (setup.source_dir / 'index.json').write_text(index_json) + if definition['identifier'] == 'helloapple' and \ + definition['type'] == 'resource': + definition['required_mappings'] = {'identifier': 'helloapple'} -default_setup = Setup() -uuidless_setup = Setup(modify_before_build=remove_all_uuids) +default_setup = lambda: Setup() +uuidless_setup = lambda: Setup(modify_before_build=remove_all_uuids) +schema_v2_setup = lambda: Setup(modify_before_build=bump_schema_v2) -def def_get(url: str) -> Response: - """Convenience wrapper for def_get()""" - return default_setup.client().get(url) +setup_makers = [default_setup, uuidless_setup, schema_v2_setup] -def test_project_url() -> None: - """Fetch index.html and verify project URL from config is present there.""" - response = def_get('/') - assert b'html' in response.data - project_url = default_setup.config()['hydrilla_project_url'] - assert escape(project_url).encode() in response.data - -@pytest.mark.parametrize('setup', [default_setup, uuidless_setup]) +@pytest.mark.usefixtures('mock_reuse') +@pytest.mark.parametrize('setup_maker', setup_makers) @pytest.mark.parametrize('item_type', ['resource', 'mapping']) -def test_get_newest(setup: Setup, item_type: str) -> None: +def test_get_newest(setup_maker, item_type: str) -> None: """ Verify that GET '/{item_type}/{item_identifier}.json' returns proper definition that is also served at: GET '/{item_type}/{item_identifier}/{item_version}' """ + setup = setup_maker() response = setup.client().get(f'/{item_type}/helloapple.json') assert response.status_code == 200 definition = json.loads(response.data.decode()) @@ -166,46 +212,58 @@ def test_get_newest(setup: Setup, item_type: str) -> None: assert response.status_code == 200 assert definition == json.loads(response.data.decode()) - assert ('uuid' in definition) == (setup is not uuidless_setup) + assert ('uuid' in definition) == (setup_maker is not uuidless_setup) hydrilla_util.validator_for(f'api_{item_type}_description-1.0.1.schema.json')\ .validate(definition) +@pytest.fixture +def setup(mock_reuse): + """Prepare server test environment in the default way.""" + return default_setup() + +def test_project_url(setup) -> None: + """Fetch index.html and verify project URL from config is present there.""" + response = setup.client().get('/') + assert b'html' in response.data + project_url = setup.config()['hydrilla_project_url'] + assert escape(project_url).encode() in response.data + @pytest.mark.parametrize('item_type', ['resource', 'mapping']) -def test_get_nonexistent(item_type: str) -> None: +def test_get_nonexistent(setup, item_type: str) -> None: """ Verify that attempts to GET a JSON definition of a nonexistent item or item version result in 404. """ - response = def_get(f'/{item_type}/nonexistentapple.json') + response = setup.client().get(f'/{item_type}/nonexistentapple.json') assert response.status_code == 404 - response = def_get(f'/{item_type}/helloapple/1.2.3.999') + response = setup.client().get(f'/{item_type}/helloapple/1.2.3.999') assert response.status_code == 404 @pytest.mark.parametrize('item_type', ['resource', 'mapping']) -def test_file_refs(item_type: str) -> None: +def test_file_refs(setup, item_type: str) -> None: """ Verify that files referenced by definitions are accessible under their proper URLs and that their hashes match. """ - response = def_get(f'/{item_type}/helloapple/2021.11.10') + response = setup.client().get(f'/{item_type}/helloapple/2021.11.10') assert response.status_code == 200 definition = json.loads(response.data.decode()) for file_ref in [*definition.get('scripts', []), *definition['source_copyright']]: hash_sum = file_ref["sha256"] - response = def_get(f'/file/sha256/{hash_sum}') + response = setup.client().get(f'/file/sha256/{hash_sum}') assert response.status_code == 200 assert sha256(response.data).digest().hex() == hash_sum -def test_empty_query() -> None: +def test_empty_query(setup) -> None: """ Verify that querying mappings for URL gives an empty list when there're no mathes. """ - response = def_get(f'/query?url=https://nonexiste.nt/example') + response = setup.client().get(f'/query?url=https://nonexiste.nt/example') assert response.status_code == 200 response_object = json.loads(response.data.decode()) @@ -219,12 +277,12 @@ def test_empty_query() -> None: hydrilla_util.validator_for('api_query_result-1.0.1.schema.json')\ .validate(response_object) -def test_query() -> None: +def test_query(setup) -> None: """ Verify that querying mappings for URL gives a list with reference(s) the the matching mapping(s). """ - response = def_get(f'/query?url=https://hydrillabugs.koszko.org/') + response = setup.client().get(f'/query?url=https://hydrillabugs.koszko.org/') assert response.status_code == 200 response_object = json.loads(response.data.decode()) @@ -239,12 +297,12 @@ def test_query() -> None: 'generated_by': expected_generated_by } - hydrilla_util.validator_for('api_query_result-1.0.1.schema.json')\ + hydrilla_util.validator_for('api_query_result-1.schema.json')\ .validate(response_object) -def test_source() -> None: +def test_source(setup) -> None: """Verify source descriptions are properly served.""" - response = def_get(f'/source/hello.json') + response = setup.client().get(f'/source/hello.json') assert response.status_code == 200 description = json.loads(response.data.decode()) @@ -254,18 +312,18 @@ def test_source() -> None: ['hello-message', 'helloapple', 'helloapple'] zipfile_hash = description['source_archives']['zip']['sha256'] - response = def_get(f'/source/hello.zip') + response = setup.client().get(f'/source/hello.zip') assert sha256(response.data).digest().hex() == zipfile_hash - hydrilla_util.validator_for('api_source_description-1.0.1.schema.json')\ + hydrilla_util.validator_for('api_source_description-1.schema.json')\ .validate(description) -def test_missing_source() -> None: +def test_missing_source(setup) -> None: """Verify requests for nonexistent sources result in 404.""" - response = def_get(f'/source/nonexistent.json') + response = setup.client().get(f'/source/nonexistent.json') assert response.status_code == 404 - response = def_get(f'/source/nonexistent.zip') + response = setup.client().get(f'/source/nonexistent.zip') assert response.status_code == 404 def test_normalize_version(): |