diff options
Diffstat (limited to 'src/hydrilla')
20 files changed, 1155 insertions, 0 deletions
diff --git a/src/hydrilla/__init__.py b/src/hydrilla/__init__.py new file mode 100644 index 0000000..6aeb276 --- /dev/null +++ b/src/hydrilla/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: 0BSD + +# Copyright (C) 2013-2020, PyPA + +# https://packaging.python.org/en/latest/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages + +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/src/hydrilla/builder/__init__.py b/src/hydrilla/builder/__init__.py new file mode 100644 index 0000000..73dc579 --- /dev/null +++ b/src/hydrilla/builder/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: CC0-1.0 + +# Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> +# +# Available under the terms of Creative Commons Zero v1.0 Universal. + +from .build import Build diff --git a/src/hydrilla/builder/__main__.py b/src/hydrilla/builder/__main__.py new file mode 100644 index 0000000..87dc9e2 --- /dev/null +++ b/src/hydrilla/builder/__main__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: CC0-1.0 + +# Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> +# +# Available under the terms of Creative Commons Zero v1.0 Universal. + +from . import build + +build.perform() diff --git a/src/hydrilla/builder/_version.py b/src/hydrilla/builder/_version.py new file mode 100644 index 0000000..d953eef --- /dev/null +++ b/src/hydrilla/builder/_version.py @@ -0,0 +1,5 @@ +# coding: utf-8 +# file generated by setuptools_scm +# don't change, don't track in version control +version = '1.0' +version_tuple = (1, 0) diff --git a/src/hydrilla/builder/build.py b/src/hydrilla/builder/build.py new file mode 100644 index 0000000..8eec4a4 --- /dev/null +++ b/src/hydrilla/builder/build.py @@ -0,0 +1,417 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +# Building Hydrilla packages. +# +# This file is part of Hydrilla +# +# Copyright (C) 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this +# file's license. Although I request that you do not make use this code +# in a proprietary program, I am not going to enforce this in court. + +# Enable using with Python 3.7. +from __future__ import annotations + +import json +import re +import zipfile +from pathlib import Path +from hashlib import sha256 +from sys import stderr + +import jsonschema +import click + +from .. import util +from . import _version + +here = Path(__file__).resolve().parent + +_ = util.translation(here / 'locales').gettext + +index_validator = util.validator_for('package_source-1.0.1.schema.json') + +schemas_root = 'https://hydrilla.koszko.org/schemas' + +generated_by = { + 'name': 'hydrilla.builder', + 'version': _version.version +} + +class FileReferenceError(Exception): + """ + Exception used to report various problems concerning files referenced from + source package's index.json. + """ + +class ReuseError(Exception): + """ + Exception used to report various problems when calling the REUSE tool. + """ + +class FileBuffer: + """ + Implement a file-like object that buffers data written to it. + """ + def __init__(self): + """ + Initialize FileBuffer. + """ + self.chunks = [] + + def write(self, b): + """ + Buffer 'b', return number of bytes buffered. + + 'b' is expected to be an instance of 'bytes' or 'str', in which case it + gets encoded as UTF-8. + """ + if type(b) is str: + b = b.encode() + self.chunks.append(b) + return len(b) + + def flush(self): + """ + A no-op mock of file-like object's flush() method. + """ + pass + + def get_bytes(self): + """ + Return all data written so far concatenated into a single 'bytes' + object. + """ + return b''.join(self.chunks) + +def generate_spdx_report(root): + """ + Use REUSE tool to generate an SPDX report for sources under 'root' and + return the report's contents as 'bytes'. + + 'root' shall be an instance of pathlib.Path. + + In case the directory tree under 'root' does not constitute a + REUSE-compliant package, linting report is printed to standard output and + an exception is raised. + + In case the reuse package is not installed, an exception is also raised. + """ + try: + from reuse._main import main as reuse_main + except ModuleNotFoundError: + raise ReuseError(_('couldnt_import_reuse_is_it_installed')) + + mocked_output = FileBuffer() + if reuse_main(args=['--root', str(root), 'lint'], out=mocked_output) != 0: + stderr.write(mocked_output.get_bytes().decode()) + raise ReuseError(_('spdx_report_from_reuse_incompliant')) + + mocked_output = FileBuffer() + if reuse_main(args=['--root', str(root), 'spdx'], out=mocked_output) != 0: + stderr.write(mocked_output.get_bytes().decode()) + raise ReuseError("Couldn't generate an SPDX report for package.") + + return mocked_output.get_bytes() + +class FileRef: + """Represent reference to a file in the package.""" + def __init__(self, path: Path, contents: bytes): + """Initialize FileRef.""" + self.include_in_distribution = False + self.include_in_zipfile = True + self.path = path + self.contents = contents + + self.contents_hash = sha256(contents).digest().hex() + + def make_ref_dict(self, filename: str): + """ + Represent the file reference through a dict that can be included in JSON + defintions. + """ + return { + 'file': filename, + 'sha256': self.contents_hash + } + +class Build: + """ + Build a Hydrilla package. + """ + def __init__(self, srcdir, index_json_path): + """ + Initialize a build. All files to be included in a distribution package + are loaded into memory, all data gets validated and all necessary + computations (e.g. preparing of hashes) are performed. + + 'srcdir' and 'index_json' are expected to be pathlib.Path objects. + """ + self.srcdir = srcdir.resolve() + self.index_json_path = index_json_path + self.files_by_path = {} + self.resource_list = [] + self.mapping_list = [] + + if not index_json_path.is_absolute(): + self.index_json_path = (self.srcdir / self.index_json_path) + + self.index_json_path = self.index_json_path.resolve() + + with open(self.index_json_path, 'rt') as index_file: + index_json_text = index_file.read() + + index_obj = json.loads(util.strip_json_comments(index_json_text)) + + self.files_by_path[self.srcdir / 'index.json'] = \ + FileRef(self.srcdir / 'index.json', index_json_text.encode()) + + self._process_index_json(index_obj) + + def _process_file(self, filename: str, include_in_distribution: bool=True): + """ + Resolve 'filename' relative to srcdir, load it to memory (if not loaded + before), compute its hash and store its information in + 'self.files_by_path'. + + 'filename' shall represent a relative path using '/' as a separator. + + if 'include_in_distribution' is True it shall cause the file to not only + be included in the source package's zipfile, but also written as one of + built package's files. + + Return file's reference object that can be included in JSON defintions + of various kinds. + """ + path = self.srcdir + for segment in filename.split('/'): + path /= segment + + path = path.resolve() + if not path.is_relative_to(self.srcdir): + raise FileReferenceError(_('loading_{}_outside_package_dir') + .format(filename)) + + if str(path.relative_to(self.srcdir)) == 'index.json': + raise FileReferenceError(_('loading_reserved_index_json')) + + file_ref = self.files_by_path.get(path) + if file_ref is None: + with open(path, 'rb') as file_handle: + contents = file_handle.read() + + file_ref = FileRef(path, contents) + self.files_by_path[path] = file_ref + + if include_in_distribution: + file_ref.include_in_distribution = True + + return file_ref.make_ref_dict(filename) + + def _prepare_source_package_zip(self, root_dir_name: str): + """ + Create and store in memory a .zip archive containing files needed to + build this source package. + + 'root_dir_name' shall not contain any slashes ('/'). + + Return zipfile's sha256 sum's hexstring. + """ + fb = FileBuffer() + root_dir_path = Path(root_dir_name) + + def zippath(file_path): + file_path = root_dir_path / file_path.relative_to(self.srcdir) + return file_path.as_posix() + + with zipfile.ZipFile(fb, 'w') as xpi: + for file_ref in self.files_by_path.values(): + if file_ref.include_in_zipfile: + xpi.writestr(zippath(file_ref.path), file_ref.contents) + + self.source_zip_contents = fb.get_bytes() + + return sha256(self.source_zip_contents).digest().hex() + + def _process_item(self, item_def: dict): + """ + Process 'item_def' as definition of a resource/mapping and store in + memory its processed form and files used by it. + + Return a minimal item reference suitable for using in source + description. + """ + copy_props = ['type', 'identifier', 'long_name', 'description'] + for prop in ('comment', 'uuid'): + if prop in item_def: + copy_props.append(prop) + + if item_def['type'] == 'resource': + item_list = self.resource_list + + copy_props.append('revision') + + script_file_refs = [self._process_file(f['file']) + for f in item_def.get('scripts', [])] + + deps = [{'identifier': res_ref['identifier']} + for res_ref in item_def.get('dependencies', [])] + + new_item_obj = { + 'dependencies': deps, + 'scripts': script_file_refs + } + else: + item_list = self.mapping_list + + payloads = {} + for pat, res_ref in item_def.get('payloads', {}).items(): + payloads[pat] = {'identifier': res_ref['identifier']} + + new_item_obj = { + 'payloads': payloads + } + + new_item_obj.update([(p, item_def[p]) for p in copy_props]) + + new_item_obj['version'] = util.normalize_version(item_def['version']) + new_item_obj['$schema'] = f'{schemas_root}/api_{item_def["type"]}_description-1.schema.json' + new_item_obj['source_copyright'] = self.copyright_file_refs + new_item_obj['source_name'] = self.source_name + new_item_obj['generated_by'] = generated_by + + item_list.append(new_item_obj) + + props_in_ref = ('type', 'identifier', 'version', 'long_name') + return dict([(prop, new_item_obj[prop]) for prop in props_in_ref]) + + def _process_index_json(self, index_obj: dict): + """ + Process 'index_obj' as contents of source package's index.json and store + in memory this source package's zipfile as well as package's individual + files and computed definitions of the source package and items defined + in it. + """ + index_validator.validate(index_obj) + + schema = f'{schemas_root}/api_source_description-1.schema.json' + + self.source_name = index_obj['source_name'] + + generate_spdx = index_obj.get('reuse_generate_spdx_report', False) + if generate_spdx: + contents = generate_spdx_report(self.srcdir) + spdx_path = (self.srcdir / 'report.spdx').resolve() + spdx_ref = FileRef(spdx_path, contents) + + spdx_ref.include_in_zipfile = False + self.files_by_path[spdx_path] = spdx_ref + + self.copyright_file_refs = \ + [self._process_file(f['file']) for f in index_obj['copyright']] + + if generate_spdx and not spdx_ref.include_in_distribution: + raise FileReferenceError(_('report_spdx_not_in_copyright_list')) + + item_refs = [self._process_item(d) for d in index_obj['definitions']] + + for file_ref in index_obj.get('additional_files', []): + self._process_file(file_ref['file'], include_in_distribution=False) + + root_dir_path = Path(self.source_name) + + source_archives_obj = { + 'zip' : { + 'sha256': self._prepare_source_package_zip(root_dir_path) + } + } + + self.source_description = { + '$schema': schema, + 'source_name': self.source_name, + 'source_copyright': self.copyright_file_refs, + 'upstream_url': index_obj['upstream_url'], + 'definitions': item_refs, + 'source_archives': source_archives_obj, + 'generated_by': generated_by + } + + if 'comment' in index_obj: + self.source_description['comment'] = index_obj['comment'] + + def write_source_package_zip(self, dstpath: Path): + """ + Create a .zip archive containing files needed to build this source + package and write it at 'dstpath'. + """ + with open(dstpath, 'wb') as output: + output.write(self.source_zip_contents) + + def write_package_files(self, dstpath: Path): + """Write package files under 'dstpath' for distribution.""" + file_dir_path = (dstpath / 'file' / 'sha256').resolve() + file_dir_path.mkdir(parents=True, exist_ok=True) + + for file_ref in self.files_by_path.values(): + if file_ref.include_in_distribution: + file_path = file_dir_path / file_ref.contents_hash + file_path.write_bytes(file_ref.contents) + + source_dir_path = (dstpath / 'source').resolve() + source_dir_path.mkdir(parents=True, exist_ok=True) + source_name = self.source_description["source_name"] + + with open(source_dir_path / f'{source_name}.json', 'wt') as output: + json.dump(self.source_description, output) + + with open(source_dir_path / f'{source_name}.zip', 'wb') as output: + output.write(self.source_zip_contents) + + for item_type, item_list in [ + ('resource', self.resource_list), + ('mapping', self.mapping_list) + ]: + item_type_dir_path = (dstpath / item_type).resolve() + + for item_def in item_list: + item_dir_path = item_type_dir_path / item_def['identifier'] + item_dir_path.mkdir(parents=True, exist_ok=True) + + version = '.'.join([str(n) for n in item_def['version']]) + with open(item_dir_path / version, 'wt') as output: + json.dump(item_def, output) + +dir_type = click.Path(exists=True, file_okay=False, resolve_path=True) + +@click.option('-s', '--srcdir', default='./', type=dir_type, show_default=True, + help=_('source_directory_to_build_from')) +@click.option('-i', '--index-json', default='index.json', type=click.Path(), + help=_('path_instead_of_index_json')) +@click.option('-d', '--dstdir', type=dir_type, required=True, + help=_('built_package_files_destination')) +@click.version_option(version=_version.version, prog_name='Hydrilla builder', + message=_('%(prog)s_%(version)s_license'), + help=_('version_printing')) +def perform(srcdir, index_json, dstdir): + """<this will be replaced by a localized docstring for Click to pick up>""" + build = Build(Path(srcdir), Path(index_json)) + build.write_package_files(Path(dstdir)) + +perform.__doc__ = _('build_package_from_srcdir_to_dstdir') + +perform = click.command()(perform) diff --git a/src/hydrilla/builder/locales/en_US/LC_MESSAGES/hydrilla-messages.po b/src/hydrilla/builder/locales/en_US/LC_MESSAGES/hydrilla-messages.po new file mode 100644 index 0000000..e3ab525 --- /dev/null +++ b/src/hydrilla/builder/locales/en_US/LC_MESSAGES/hydrilla-messages.po @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: CC0-1.0 +# +# English (United States) translations for hydrilla.builder. +# Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> +# Available under the terms of Creative Commons Zero v1.0 Universal. +msgid "" +msgstr "" +"Project-Id-Version: hydrilla.builder 0.1.dev16+g4e46d7f.d20220211\n" +"Report-Msgid-Bugs-To: koszko@koszko.org\n" +"POT-Creation-Date: 2022-04-19 13:51+0200\n" +"PO-Revision-Date: 2022-02-12 00:00+0000\n" +"Last-Translator: Wojtek Kosior <koszko@koszko.org>\n" +"Language: en_US\n" +"Language-Team: en_US <koszko@koszko.org>\n" +"Plural-Forms: nplurals=2; plural=(n != 1)\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.8.0\n" + +#: src/hydrilla/builder/build.py:118 +msgid "couldnt_import_reuse_is_it_installed" +msgstr "" +"Could not import 'reuse'. Is the tool installed and visible to this " +"Python instance?" + +#: src/hydrilla/builder/build.py:123 +msgid "spdx_report_from_reuse_incompliant" +msgstr "Attempt to generate an SPDX report for a REUSE-incompliant package." + +#: src/hydrilla/builder/build.py:207 +msgid "loading_{}_outside_package_dir" +msgstr "Attempt to load '{}' which lies outside package source directory." + +#: src/hydrilla/builder/build.py:211 +msgid "loading_reserved_index_json" +msgstr "Attempt to load 'index.json' which is a reserved filename." + +#: src/hydrilla/builder/build.py:329 +msgid "report_spdx_not_in_copyright_list" +msgstr "" +"Told to generate 'report.spdx' but 'report.spdx' is not listed among " +"copyright files. Refusing to proceed." + +#: src/hydrilla/builder/build.py:402 +msgid "source_directory_to_build_from" +msgstr "Source directory to build from." + +#: src/hydrilla/builder/build.py:404 +msgid "path_instead_of_index_json" +msgstr "" +"Path to file to be processed instead of index.json (if not absolute, " +"resolved relative to srcdir)." + +#: src/hydrilla/builder/build.py:406 +msgid "built_package_files_destination" +msgstr "Destination directory to write built package files to." + +#: src/hydrilla/builder/build.py:408 +#, python-format +msgid "%(prog)s_%(version)s_license" +msgstr "" +"%(prog)s %(version)s\n" +"Copyright (C) 2021,2022 Wojtek Kosior and contributors.\n" +"License GPLv3+: GNU AGPL version 3 or later " +"<https://gnu.org/licenses/gpl.html>\n" +"This is free software: you are free to change and redistribute it.\n" +"There is NO WARRANTY, to the extent permitted by law." + +#: src/hydrilla/builder/build.py:409 +msgid "version_printing" +msgstr "Print version information and exit." + +#: src/hydrilla/builder/build.py:415 +msgid "build_package_from_srcdir_to_dstdir" +msgstr "" +"Build Hydrilla package from `scrdir` and write the resulting files under " +"`dstdir`." + +#: src/hydrilla/util/_util.py:79 +msgid "bad_comment" +msgstr "bad comment" + diff --git a/src/hydrilla/schemas/api_mapping_description-1.0.1.schema.json b/src/hydrilla/schemas/api_mapping_description-1.0.1.schema.json new file mode 100644 index 0000000..880a5c4 --- /dev/null +++ b/src/hydrilla/schemas/api_mapping_description-1.0.1.schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://hydrilla.koszko.org/schemas/api_mapping_description-1.0.1.schema.json", + "title": "Mapping description", + "description": "Definition of a Hydrilla mapping, as served through HTTP API", + "allOf": [{ + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/mapping_definition_base" + }, { + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/item_definition" + }, { + "type": "object", + "required": ["$schema"], + "properties": { + "$schema": { + "description": "Mark this instance as conforming to mapping description schema 1.x", + "type": "string", + "pattern": "^https://hydrilla\\.koszko\\.org/schemas/api_mapping_description-1\\.(([1-9][0-9]*|0)\\.)*schema\\.json$" + } + } + }] +} diff --git a/src/hydrilla/schemas/api_mapping_description-1.0.1.schema.json.license b/src/hydrilla/schemas/api_mapping_description-1.0.1.schema.json.license new file mode 100644 index 0000000..f41d511 --- /dev/null +++ b/src/hydrilla/schemas/api_mapping_description-1.0.1.schema.json.license @@ -0,0 +1,5 @@ +SPDX-License-Identifier: CC0-1.0 + +Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> + +Available under the terms of Creative Commons Zero v1.0 Universal. diff --git a/src/hydrilla/schemas/api_query_result-1.0.1.schema.json b/src/hydrilla/schemas/api_query_result-1.0.1.schema.json new file mode 100644 index 0000000..89c5428 --- /dev/null +++ b/src/hydrilla/schemas/api_query_result-1.0.1.schema.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://hydrilla.koszko.org/schemas/api_query_result-1.0.1.schema.json", + "title": "Query result", + "description": "Object with a list of references to mappings that contain payloads for requested URL", + "type": "object", + "required": ["$schema", "mappings"], + "properties": { + "$schema": { + "description": "Mark this instance as conforming to query result schema 1.x", + "type": "string", + "pattern": "^https://hydrilla\\.koszko\\.org/schemas/api_query_result-1\\.(([1-9][0-9]*|0)\\.)*schema\\.json$" + }, + "mappings": { + "description": "References to mappings using at least one pattern that matches the requested URL", + "type": "array", + "items": { + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/item_ref" + } + }, + "generated_by": { + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/generated_by" + } + } +} diff --git a/src/hydrilla/schemas/api_query_result-1.0.1.schema.json.license b/src/hydrilla/schemas/api_query_result-1.0.1.schema.json.license new file mode 100644 index 0000000..f41d511 --- /dev/null +++ b/src/hydrilla/schemas/api_query_result-1.0.1.schema.json.license @@ -0,0 +1,5 @@ +SPDX-License-Identifier: CC0-1.0 + +Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> + +Available under the terms of Creative Commons Zero v1.0 Universal. diff --git a/src/hydrilla/schemas/api_resource_description-1.0.1.schema.json b/src/hydrilla/schemas/api_resource_description-1.0.1.schema.json new file mode 100644 index 0000000..7459394 --- /dev/null +++ b/src/hydrilla/schemas/api_resource_description-1.0.1.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://hydrilla.koszko.org/schemas/api_resource_description-1.0.1.schema.json", + "title": "Resource description", + "description": "Definition of a Hydrilla resource, as served through HTTP API", + "allOf": [{ + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/resource_definition_base" + }, { + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/item_definition" + }, { + "type": "object", + "required": ["$schema"], + "properties": { + "$schema": { + "description": "Mark this instance as conforming to resource description schema 1.x", + "type": "string", + "pattern": "^https://hydrilla\\.koszko\\.org/schemas/api_resource_description-1\\.(([1-9][0-9]*|0)\\.)*schema\\.json$" + }, + "scripts": { + "description": "Which files are resource's scripts and need to be installed", + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/file_ref_list_sha256", + "default": [] + } + } + }] +} diff --git a/src/hydrilla/schemas/api_resource_description-1.0.1.schema.json.license b/src/hydrilla/schemas/api_resource_description-1.0.1.schema.json.license new file mode 100644 index 0000000..f41d511 --- /dev/null +++ b/src/hydrilla/schemas/api_resource_description-1.0.1.schema.json.license @@ -0,0 +1,5 @@ +SPDX-License-Identifier: CC0-1.0 + +Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> + +Available under the terms of Creative Commons Zero v1.0 Universal. diff --git a/src/hydrilla/schemas/api_source_description-1.0.1.schema.json b/src/hydrilla/schemas/api_source_description-1.0.1.schema.json new file mode 100644 index 0000000..0744d1a --- /dev/null +++ b/src/hydrilla/schemas/api_source_description-1.0.1.schema.json @@ -0,0 +1,66 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://hydrilla.koszko.org/schemas/api_source_description-1.0.1.schema.json", + "title": "Source description", + "description": "Built description of a Hydrilla source package", + "type": "object", + "required": [ + "$schema", + "source_name", + "source_copyright", + "source_archives", + "upstream_url", + "definitions" + ], + "properties": { + "$schema": { + "description": "Mark this instance as conforming to source description schema 1.x", + "type": "string", + "pattern": "^https://hydrilla\\.koszko\\.org/schemas/api_source_description-1\\.(([1-9][0-9]*|0)\\.)*schema\\.json$" + }, + "source_name": { + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/source_name" + }, + "source_copyright": { + "description": "Which files indicate license terms of the source package", + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/file_ref_list_sha256" + }, + "source_archives": { + "description": "What archive extensions are available for this package's sources", + "type": "object", + "required": ["zip"], + "additionalProperties": { + "description": "What is the SHA256 sum of given source archive", + "type": "object", + "required": ["sha256"], + "properties": { + "sha256": { + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/sha256" + } + } + }, + "examples": [{ + "zip": { + "sha256": "688461da362ffe2fc8e85db73e709a5356d41c8aeb7d1eee7170c64ee21dd2a2" + } + }] + }, + "upstream_url": { + "description": "Where this software/work initially comes from", + "type": "string" + }, + "comment": { + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/comment" + }, + "definitions": { + "description": "References to site resources and pattern->payload mappings", + "type": "array", + "items": { + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/typed_item_ref" + } + }, + "generated_by": { + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/generated_by" + } + } +} diff --git a/src/hydrilla/schemas/api_source_description-1.0.1.schema.json.license b/src/hydrilla/schemas/api_source_description-1.0.1.schema.json.license new file mode 100644 index 0000000..f41d511 --- /dev/null +++ b/src/hydrilla/schemas/api_source_description-1.0.1.schema.json.license @@ -0,0 +1,5 @@ +SPDX-License-Identifier: CC0-1.0 + +Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> + +Available under the terms of Creative Commons Zero v1.0 Universal. diff --git a/src/hydrilla/schemas/common_definitions-1.0.1.schema.json b/src/hydrilla/schemas/common_definitions-1.0.1.schema.json new file mode 100644 index 0000000..b803188 --- /dev/null +++ b/src/hydrilla/schemas/common_definitions-1.0.1.schema.json @@ -0,0 +1,233 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://hydrilla.koszko.org/schemas/common_definitions-1.0.1.schema.json", + "title": "Common definitions", + "description": "Definitions used by other Hydrilla schemas", + "definitions": { + "version": { + "description": "Version expressed as an array of integers", + "type": "array", + "minItems": 1, + "items": { + "type": "integer", + "minimum": 0 + }, + "contains": { + "type": "integer", + "minimum": 1 + }, + "minItems": 1 + }, + "source_name": { + "description": "Unique identifier of this source package", + "type": "string", + "pattern": "^[-0-9a-z.]+$" + }, + "comment": { + "description": "An optional comment", + "type": "string" + }, + "file_ref_list": { + "description": "List of simple file references", + "type": "array", + "items": { + "type": "object", + "required": ["file"], + "properties": { + "file": { + "description": "Filename relative to source package main directory; separator is '/'", + "type": "string", + "pattern": "^[^/]" + } + } + } + }, + "sha256": { + "description": "An SHA256 sum, in hexadecimal", + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "file_ref_list_sha256": { + "description": "List of file references with files' SHA256 sums included", + "allOf": [{ + "$ref": "#/definitions/file_ref_list" + }, { + "type": "array", + "items": { + "type": "object", + "required": ["sha256"], + "properties": { + "sha256": { + "$ref": "#/definitions/sha256" + } + } + } + }] + }, + "item_identifier": { + "description": "Identifier of an item (shared with other versions of the item, otherwise unique)", + "type": "string", + "pattern": "^[-0-9a-z]+$" + }, + "item_dep_specifier": { + "description": "Simple reference to an item as a dependency", + "type": "object", + "required": ["identifier"], + "properties": { + "identifier": { + "$ref": "#/definitions/item_identifier" + } + } + }, + "item_ref": { + "description": "An object containing a subset of fields from full item definition", + "type": "object", + "required": ["identifier", "long_name", "version"], + "properties": { + "identifier": { + "$ref": "#/definitions/item_identifier" + }, + "long_name": { + "description": "User-friendly alternative to the identifier", + "type": "string" + }, + "version": { + "$ref": "#/definitions/version" + } + } + }, + "typed_item_ref": { + "description": "An object containing a subset of fields from full item definition, including type", + "allOf": [{ + "$ref": "#/definitions/item_ref" + }, { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "description": "What kind of item is it (resource or mapping)", + "enum": ["resource", "mapping"] + } + } + }] + }, + "item_definition_base": { + "description": "Definition of a resource/mapping (fields common to source definitions and built definitions)", + "allOf": [{ + "$ref": "#/definitions/typed_item_ref" + }, { + "type": "object", + "required": ["description"], + "properties": { + "uuid": { + "description": "UUIDv4 of this item (shared with other versions of this item, otherwise unique)", + "type": "string", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + }, + "description": { + "description": "Item's description", + "type": "string" + }, + "comment": { + "$ref": "#/definitions/comment" + } + } + }] + }, + "resource_definition_base": { + "description": "Definition of a resource (fields common to source definitions and built definitions)", + "allOf": [{ + "$ref": "#/definitions/item_definition_base" + }, { + "type": "object", + "required": ["type", "revision"], + "properties": { + "type": { + "description": "Identify this item as a resource", + "const": "resource" + }, + "revision": { + "description": "Which revision of a packaging of given version of an upstream resource is this", + "type": "integer", + "minimum": 1 + }, + "scripts": { + "description": "What scripts are included in the resource", + "$ref": "#/definitions/file_ref_list", + "default": [] + }, + "dependencies": { + "description": "Which other resources this resource depends on", + "type": "array", + "items": { + "$ref": "#/definitions/item_dep_specifier" + }, + "default": [] + } + } + }] + }, + "mapping_definition_base": { + "description": "Definition of a mapping (fields common to source definitions and built definitions)", + "allOf": [{ + "$ref": "#/definitions/item_definition_base" + }, { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "description": "Identify this item as a mapping", + "const": "mapping" + }, + "payloads": { + "description": "Which payloads are to be applied to which URLs", + "additionalProperties": { + "$ref": "#/definitions/item_dep_specifier" + }, + "default": {}, + "examples": [{ + "https://hydrillabugs.koszko.org/***": { + "identifier": "helloapple" + }, + "https://*.koszko.org/***": { + "identifier": "hello-potato" + } + }] + } + } + }] + }, + "generated_by": { + "description": "Describe what software generated this instance", + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string", + "description": "Instance generator software name, without version" + }, + "version": { + "type": "string", + "description": "Instance generator software version, in arbitrary format" + } + } + }, + "item_definition": { + "description": "Definition of a resource/mapping (fields specific to built definitions)", + "type": "object", + "required": ["source_name", "source_copyright"], + "properties": { + "source_name": { + "$ref": "#/definitions/source_name" + }, + "source_copyright": { + "description": "Which files indicate license terms of the source package and should be installed", + "$ref": "#/definitions/file_ref_list_sha256" + }, + "generated_by": { + "$ref": "#/definitions/generated_by" + } + } + } + } +} diff --git a/src/hydrilla/schemas/common_definitions-1.0.1.schema.json.license b/src/hydrilla/schemas/common_definitions-1.0.1.schema.json.license new file mode 100644 index 0000000..f41d511 --- /dev/null +++ b/src/hydrilla/schemas/common_definitions-1.0.1.schema.json.license @@ -0,0 +1,5 @@ +SPDX-License-Identifier: CC0-1.0 + +Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> + +Available under the terms of Creative Commons Zero v1.0 Universal. diff --git a/src/hydrilla/schemas/package_source-1.0.1.schema.json b/src/hydrilla/schemas/package_source-1.0.1.schema.json new file mode 100644 index 0000000..2f9482e --- /dev/null +++ b/src/hydrilla/schemas/package_source-1.0.1.schema.json @@ -0,0 +1,56 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://hydrilla.koszko.org/schemas/package_source-1.0.1.schema.json", + "title": "Package source", + "description": "Definition of a Hydrilla source package", + "type": "object", + "required": [ + "$schema", + "source_name", + "copyright", + "upstream_url", + "definitions" + ], + "properties": { + "$schema": { + "description": "Mark this instance as conforming to package source schema 1.x", + "type": "string", + "pattern": "^https://hydrilla\\.koszko\\.org/schemas/package_source-1\\.(([1-9][0-9]*|0)\\.)*schema\\.json$" + }, + "source_name": { + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/source_name" + }, + "copyright": { + "description": "Which files from the source package indicate its license terms and should be included in the distribution packages", + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/file_ref_list" + }, + "upstream_url": { + "description": "Where this software/work initially comes from", + "type": "string" + }, + "comment": { + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/comment" + }, + "definitions": { + "description": "Definitions of site resources and pattern->payload mappings", + "type": "array", + "items": { + "anyOf": [{ + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/resource_definition_base" + }, { + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/mapping_definition_base" + }] + } + }, + "additional_files": { + "description": "Files which should be included in the source archive produced by Hydrilla builder in addition to script and copyright files", + "$ref": "./common_definitions-1.0.1.schema.json#/definitions/file_ref_list", + "default": [] + }, + "reuse_generate_spdx_report": { + "description": "Should report.spdx be automatically generated for the package using REUSE tool", + "type": "boolean", + "default": false + } + } +} diff --git a/src/hydrilla/schemas/package_source-1.0.1.schema.json.license b/src/hydrilla/schemas/package_source-1.0.1.schema.json.license new file mode 100644 index 0000000..f41d511 --- /dev/null +++ b/src/hydrilla/schemas/package_source-1.0.1.schema.json.license @@ -0,0 +1,5 @@ +SPDX-License-Identifier: CC0-1.0 + +Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> + +Available under the terms of Creative Commons Zero v1.0 Universal. diff --git a/src/hydrilla/util/__init__.py b/src/hydrilla/util/__init__.py new file mode 100644 index 0000000..fadb81c --- /dev/null +++ b/src/hydrilla/util/__init__.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: CC0-1.0 + +# Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> +# +# Available under the terms of Creative Commons Zero v1.0 Universal. + +from ._util import strip_json_comments, normalize_version, parse_version, \ + version_string, validator_for, translation diff --git a/src/hydrilla/util/_util.py b/src/hydrilla/util/_util.py new file mode 100644 index 0000000..778e78f --- /dev/null +++ b/src/hydrilla/util/_util.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +# Building Hydrilla packages. +# +# This file is part of Hydrilla +# +# Copyright (C) 2021, 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this +# file's license. Although I request that you do not make use this code +# in a proprietary program, I am not going to enforce this in court. + +# Enable using with Python 3.7. +from __future__ import annotations + +import re +import json +import locale +import gettext + +from pathlib import Path +from typing import Optional, Union + +from jsonschema import RefResolver, Draft7Validator + +here = Path(__file__).resolve().parent + +_strip_comment_re = re.compile(r''' +^ # match from the beginning of each line +( # catch the part before '//' comment + (?: # this group matches either a string or a single out-of-string character + [^"/] | + " + (?: # this group matches any in-a-string character + [^"\\] | # match any normal character + \\[^u] | # match any escaped character like '\f' or '\n' + \\u[a-fA-F0-9]{4} # match an escape + )* + " + )* +) +# expect either end-of-line or a comment: +# * unterminated strings will cause matching to fail +# * bad comment (with '/' instead of '//') will be indicated by second group +# having length 1 instead of 2 or 0 +(//?|$) +''', re.VERBOSE) + +def strip_json_comments(text: str) -> str: + """ + Accept JSON text with optional C++-style ('//') comments and return the text + with comments removed. Consecutive slashes inside strings are handled + properly. A spurious single slash ('/') shall generate an error. Errors in + JSON itself shall be ignored. + """ + processed = 0 + stripped_text = [] + for line in text.split('\n'): + match = _strip_comment_re.match(line) + + if match is None: # unterminated string + # ignore this error, let json module report it + stripped = line + elif len(match[2]) == 1: + raise json.JSONDecodeError(_('bad_comment'), text, + processed + len(match[1])) + else: + stripped = match[1] + + stripped_text.append(stripped) + processed += len(line) + 1 + + return '\n'.join(stripped_text) + +def normalize_version(ver: list[int]) -> list[int]: + """Strip right-most zeroes from 'ver'. The original list is not modified.""" + new_len = 0 + for i, num in enumerate(ver): + if num != 0: + new_len = i + 1 + + return ver[:new_len] + +def parse_version(ver_str: str) -> list[int]: + """ + Convert 'ver_str' into an array representation, e.g. for ver_str="4.6.13.0" + return [4, 6, 13, 0]. + """ + return [int(num) for num in ver_str.split('.')] + +def version_string(ver: list[int], rev: Optional[int]=None) -> str: + """ + Produce version's string representation (optionally with revision), like: + 1.2.3-5 + No version normalization is performed. + """ + return '.'.join([str(n) for n in ver]) + ('' if rev is None else f'-{rev}') + +schemas = {} +for path in (here.parent / 'schemas').glob('*-1.0.1.schema.json'): + schema = json.loads(path.read_text()) + schemas[schema['$id']] = schema + +common_schema_filename = 'common_definitions-1.schema.json' +common_schema_path = here.parent / "schemas" / common_schema_filename + +resolver = RefResolver( + base_uri=f'file://{str(common_schema_path)}', + referrer=f'https://hydrilla.koszko.org/{common_schema_filename}', + store=schemas +) + +def validator_for(schema_filename: str) -> Draft7Validator: + """ + Prepare a validator for one of the schemas in '../schemas'. + + This function is not thread-safe. + """ + return Draft7Validator(resolver.resolve(schema_filename)[1], + resolver=resolver) + +def translation(localedir: Union[Path, str], lang: Optional[str]=None) \ + -> gettext.GNUTranslations: + """ + Configure translations for domain 'hydrilla-messages' and return the object + that represents them. + + If `lang` is set, look for translations for `lang`. Otherwise, try to + determine system's default language and use that. + """ + # https://stackoverflow.com/questions/3425294/how-to-detect-the-os-default-language-in-python + # But I am not going to surrender to Microbugs' nonfree, crappy OS to test + # it, to the lines inside try: may fail. + if lang is None: + try: + from ctypes.windll import kernel32 as windll + lang = locale.windows_locale[windll.GetUserDefaultUILanguage()] + except: + lang = locale.getdefaultlocale()[0] or 'en_US' + + localedir = Path(localedir) + if not (localedir / lang).is_dir(): + lang = 'en_US' + + return gettext.translation('hydrilla-messages', localedir=localedir, + languages=[lang]) + +_ = translation(here.parent / 'builder' / 'locales').gettext |