summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/hydrilla_builder/build.py14
m---------src/hydrilla_builder/schemas0
-rw-r--r--src/test/test_hydrilla_builder.py419
3 files changed, 279 insertions, 154 deletions
diff --git a/src/hydrilla_builder/build.py b/src/hydrilla_builder/build.py
index dcd7e01..e7c6962 100644
--- a/src/hydrilla_builder/build.py
+++ b/src/hydrilla_builder/build.py
@@ -223,8 +223,8 @@ class Build:
index_obj = json.loads(strip_json_comments(index_json_text))
- self.files_by_path[self.index_json_path] = \
- FileRef(self.index_json_path, index_json_text.encode())
+ self.files_by_path[self.srcdir / 'index.json'] = \
+ FileRef(self.srcdir / 'index.json', index_json_text.encode())
self._process_index_json(index_obj)
@@ -302,7 +302,7 @@ class Build:
"""
copy_props = ['type', 'identifier', 'long_name', 'uuid', 'description']
if 'comment' in item_def:
- copy_props.append(copy_props)
+ copy_props.append('comment')
if item_def['type'] == 'resource':
item_list = self.resource_list
@@ -319,8 +319,9 @@ class Build:
else:
item_list = self.mapping_list
- payloads = dict([(pat, {'identifier': res_ref['identifier']})
- for pat, res_ref in item_def['payloads'].items()])
+ payloads = {}
+ for pat, res_ref in item_def.get('payloads', {}).items():
+ payloads[pat] = {'identifier': res_ref['identifier']}
new_item_obj = {
'payloads': payloads
@@ -386,6 +387,9 @@ class Build:
'source_archives': source_archives_obj
}
+ if 'comment' in index_obj:
+ self.source_description['comment'] = index_obj['comment']
+
def write_source_package_zip(self, dstpath: Path):
"""
Create a .zip archive containing files needed to build this source
diff --git a/src/hydrilla_builder/schemas b/src/hydrilla_builder/schemas
-Subproject 7ce7ee1a4821b23909c07491798083c4b3639f9
+Subproject ca1de2ed4a69a71f2f75552ade693d04ea1baa8
diff --git a/src/test/test_hydrilla_builder.py b/src/test/test_hydrilla_builder.py
index 52ed650..b2f19f0 100644
--- a/src/test/test_hydrilla_builder.py
+++ b/src/test/test_hydrilla_builder.py
@@ -14,20 +14,257 @@ from zipfile import ZipFile
here = Path(__file__).resolve().parent
+default_srcdir = here / 'source-package-example'
+
+default_js_filenames = ['bye.js', 'hello.js', 'message.js']
+default_dist_filenames = [*default_js_filenames, 'LICENSES/CC0-1.0.txt']
+default_src_filenames = [
+ *default_dist_filenames,
+ 'README.txt', 'README.txt.license', '.reuse/dep5', 'index.json'
+]
+
+default_sha1_hashes = {}
+default_sha256_hashes = {}
+default_contents = {}
+
+for fn in default_src_filenames:
+ with open(default_srcdir / fn, 'rb') as file_handle:
+ default_contents[fn] = file_handle.read()
+ default_sha256_hashes[fn] = sha256(default_contents[fn]).digest().hex()
+ default_sha1_hashes[fn] = sha1(default_contents[fn]).digest().hex()
+
+class CaseSettings:
+ """Gather parametrized values in a class."""
+ def __init__(self):
+ """Init CaseSettings with default values."""
+ self.srcdir = default_srcdir
+ self.index_json_path = Path('index.json')
+ self.report_spdx_included = True
+
+ self.js_filenames = default_js_filenames.copy()
+ self.dist_filenames = default_dist_filenames.copy()
+ self.src_filenames = default_src_filenames.copy()
+
+ self.sha1_hashes = default_sha1_hashes.copy()
+ self.sha256_hashes = default_sha256_hashes.copy()
+ self.contents = default_contents.copy()
+
+ self.expected_resources = [{
+ 'api_schema_version': [1, 0, 1],
+ 'source_name': 'hello',
+ 'source_copyright': [{
+ 'file': 'report.spdx',
+ 'sha256': '!!!!value to fill during test!!!!'
+ }, {
+ 'file': 'LICENSES/CC0-1.0.txt',
+ 'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
+ }],
+ 'type': 'resource',
+ 'identifier': 'helloapple',
+ 'long_name': 'Hello Apple',
+ 'uuid': 'a6754dcb-58d8-4b7a-a245-24fd7ad4cd68',
+ 'version': [2021, 11, 10],
+ 'revision': 1,
+ 'description': 'greets an apple',
+ 'dependencies': ['hello-message'],
+ 'scripts': [{
+ 'file': 'hello.js',
+ 'sha256': self.sha256_hashes['hello.js']
+ }, {
+ 'file': 'bye.js',
+ 'sha256': self.sha256_hashes['bye.js']
+ }]
+ }, {
+ 'api_schema_version': [1, 0, 1],
+ 'source_name': 'hello',
+ 'source_copyright': [{
+ 'file': 'report.spdx',
+ 'sha256': '!!!!value to fill during test!!!!'
+ }, {
+ 'file': 'LICENSES/CC0-1.0.txt',
+ 'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
+ }],
+ 'type': 'resource',
+ 'identifier': 'hello-message',
+ 'long_name': 'Hello Message',
+ 'uuid': '1ec36229-298c-4b35-8105-c4f2e1b9811e',
+ 'version': [2021, 11, 10],
+ 'revision': 2,
+ 'description': 'define messages for saying hello and bye',
+ 'dependencies': [],
+ 'scripts': [{
+ 'file': 'message.js',
+ 'sha256': self.sha256_hashes['message.js']
+ }]
+ }]
+ self.expected_mapping = {
+ 'api_schema_version': [1, 0, 1],
+ 'source_name': 'hello',
+ 'source_copyright': [{
+ 'file': 'report.spdx',
+ 'sha256': '!!!!value to fill during test!!!!'
+ }, {
+ 'file': 'LICENSES/CC0-1.0.txt',
+ 'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
+ }],
+ 'type': 'mapping',
+ 'identifier': 'helloapple',
+ 'long_name': 'Hello Apple',
+ 'uuid': '54d23bba-472e-42f5-9194-eaa24c0e3ee7',
+ 'version': [2021, 11, 10],
+ 'description': 'causes apple to get greeted on Hydrillabugs issue tracker',
+ 'payloads': {
+ 'https://hydrillabugs.koszko.org/***': {
+ 'identifier': 'helloapple'
+ },
+ 'https://hachettebugs.koszko.org/***': {
+ 'identifier': 'helloapple'
+ }
+ }
+ }
+ self.expected_source_description = {
+ 'api_schema_version': [1, 0, 1],
+ 'source_name': 'hello',
+ 'source_copyright': [{
+ 'file': 'report.spdx',
+ 'sha256': '!!!!value to fill during test!!!!'
+ }, {
+ 'file': 'LICENSES/CC0-1.0.txt',
+ 'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
+ }],
+ 'source_archives': {
+ 'zip': {
+ 'sha256': '!!!!value to fill during test!!!!',
+ }
+ },
+ 'upstream_url': 'https://git.koszko.org/hydrilla-source-package-example',
+ 'definitions': [{
+ 'type': 'resource',
+ 'identifier': 'helloapple',
+ 'version': [2021, 11, 10],
+ }, {
+ 'type': 'resource',
+ 'identifier': 'hello-message',
+ 'version': [2021, 11, 10],
+ }, {
+ 'type': 'mapping',
+ 'identifier': 'helloapple',
+ 'version': [2021, 11, 10],
+ }]
+ }
+
+ def expected(self) -> list[dict]:
+ """
+ Convenience method to get a list of expected jsons of 2 resources,
+ 1 mapping and 1 source description we have.
+ """
+ return [
+ *self.expected_resources,
+ self.expected_mapping,
+ self.expected_source_description
+ ]
+
@pytest.fixture()
-def tmpdir():
+def tmpdir() -> str:
with TemporaryDirectory() as tmpdir:
yield tmpdir
-def test_build(tmpdir):
+def prepare_default(tmpdir: Path) -> CaseSettings:
+ """Use sample source package directory as exists in VCS."""
+ return CaseSettings()
+
+def prepare_external_index_json(tmpdir: Path) -> dict:
+ """
+ Use sample source package directory with an alternative, modified
+ index.json.
+ """
+ settings = CaseSettings()
+
+ from hydrilla_builder.build import strip_json_comments
+
+ with open(settings.srcdir / 'index.json', 'rt') as file_handle:
+ obj = json.loads(strip_json_comments(file_handle.read()))
+
+ # Add comments that should be preserved.
+ for dictionary in (obj, settings.expected_source_description):
+ dictionary['comment'] = 'index_json comment'
+
+ for i, dicts in enumerate(zip(obj['definitions'], settings.expected())):
+ for dictionary in dicts:
+ dictionary['comment'] = f'item {i}'
+
+ # Remove spdx report generation
+ del obj['reuse_generate_spdx_report']
+ obj['copyright'].remove({'file': 'report.spdx'})
+
+ settings.report_spdx_included = False
+
+ for json_description in settings.expected():
+ json_description['source_copyright'] = \
+ [fr for fr in json_description['source_copyright']
+ if fr['file'] != 'report.spdx']
+
+ # Use default value ([]) for 'additionall_files' property
+ del obj['additional_files']
+
+ settings.src_filenames = [*settings.dist_filenames, 'index.json']
+
+ # Use default value ([]) for 'scripts' property in one of the resources
+ del obj['definitions'][1]['scripts']
+
+ settings.expected_resources[1]['scripts'] = []
+
+ for prefix in ('js', 'dist', 'src'):
+ getattr(settings, f'{prefix}_filenames').remove('message.js')
+
+ # Use default value ({}) for 'pyloads' property in mapping
+ del obj['definitions'][2]['payloads']
+
+ settings.expected_mapping['payloads'] = {}
+
+ # Add some unrecognized properties that should be stripped
+ to_process = [obj]
+ while to_process:
+ processed = to_process.pop()
+
+ if type(processed) is list:
+ to_process.extend(processed)
+ elif type(processed) is dict and 'spurious_property' not in processed:
+ to_process.extend(processed.values())
+ processed['spurious_property'] = 'some value'
+
+ # Replace the other index.json with new one
+ settings.index_json_path = tmpdir / 'replacement.json'
+
+ contents = json.dumps(obj).encode()
+
+ settings.contents['index.json'] = contents
+
+ settings.sha256_hashes['index.json'] = sha256(contents).digest().hex()
+ settings.sha1_hashes['index.json'] = sha1(contents).digest().hex()
+
+ with open(settings.index_json_path, 'wb') as file_handle:
+ file_handle.write(contents)
+
+ return settings
+
+@pytest.mark.parametrize('prepare_source_example', [
+ prepare_default, prepare_external_index_json
+])
+def test_build(tmpdir, prepare_source_example):
"""Build the sample source package and verify the produced files."""
from hydrilla_builder.build import Build
# First, build the package
- srcdir = here / 'source-package-example'
- dstdir = Path(tmpdir)
+ dstdir = Path(tmpdir) / 'dstdir'
+ tmpdir = Path(tmpdir) / 'example'
+
+ dstdir.mkdir(exist_ok=True)
+ tmpdir.mkdir(exist_ok=True)
- build = Build(srcdir, Path('index.json'))
+ settings = prepare_source_example(tmpdir)
+
+ build = Build(settings.srcdir, settings.index_json_path)
build.write_package_files(dstdir)
# Verify directories under destination directory
@@ -36,35 +273,25 @@ def test_build(tmpdir):
# Verify files under 'file/'
file_dir = dstdir / 'file'
- js_filenames = ('bye.js', 'hello.js', 'message.js')
- dist_filenames = (*js_filenames, str(Path('LICENSES') / 'CC0-1.0.txt'))
- file_contents = {}
- file_sha256_hashes = {}
- file_sha1_hashes = {}
-
- for fn in dist_filenames:
- with open(srcdir / fn, 'rb') as file_handle:
- file_contents = file_handle.read()
- file_sha256_hashes[fn] = sha256(file_contents).digest().hex()
- file_sha1_hashes[fn] = sha1(file_contents).digest().hex()
-
- dist_file_path = file_dir / f'sha256-{file_sha256_hashes[fn]}'
+ for fn in settings.dist_filenames:
+ dist_file_path = file_dir / f'sha256-{settings.sha256_hashes[fn]}'
assert dist_file_path.is_file()
with open(dist_file_path, 'rb') as file_handle:
- assert file_handle.read() == file_contents
+ assert file_handle.read() == settings.contents[fn]
+
+ sha256_hashes_set = set([settings.sha256_hashes[fn]
+ for fn in settings.dist_filenames])
- sha256_hashes = set(file_sha256_hashes.values())
spdx_report_sha256 = None
- spdx_report_checked = False
for path in file_dir.iterdir():
assert path.name.startswith('sha256-')
- if path.name[7:] in sha256_hashes:
+ if path.name[7:] in sha256_hashes_set:
continue
- assert spdx_report_sha256 is None
+ assert spdx_report_sha256 is None and settings.report_spdx_included
with open(path, 'rt') as file_handle:
spdx_contents = file_handle.read()
@@ -72,65 +299,24 @@ def test_build(tmpdir):
spdx_report_sha256 = sha256(spdx_contents.encode()).digest().hex()
assert spdx_report_sha256 == path.name[7:]
- for fn in js_filenames:
- assert file_sha1_hashes[fn] in spdx_contents
+ for fn in settings.src_filenames:
+ if not any([n in fn.lower() for n in ('license', 'reuse')]):
+ assert settings.sha1_hashes[fn]
+
+ if settings.report_spdx_included:
+ assert spdx_report_sha256
+ for obj in settings.expected():
+ for file_ref in obj['source_copyright']:
+ if file_ref['file'] == 'report.spdx':
+ file_ref['sha256'] = spdx_report_sha256
# Verify files under 'resource/'
resource_dir = dstdir / 'resource'
- expected_resource_jsons = [{
- 'api_schema_version': [1, 0, 1],
- 'source_name': 'hello',
- 'source_copyright': [{
- 'file': 'report.spdx',
- 'sha256': spdx_report_sha256
- }, {
- 'file': 'LICENSES/CC0-1.0.txt',
- 'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt']
- }],
- 'type': 'resource',
- 'identifier': 'helloapple',
- 'long_name': 'Hello Apple',
- 'uuid': 'a6754dcb-58d8-4b7a-a245-24fd7ad4cd68',
- 'version': [2021, 11, 10],
- 'revision': 1,
- 'description': 'greets an apple',
- 'dependencies': ['hello-message'],
- 'scripts': [{
- 'file': 'hello.js',
- 'sha256': file_sha256_hashes['hello.js']
- }, {
- 'file': 'bye.js',
- 'sha256': file_sha256_hashes['bye.js']
- }]
- }, {
- 'api_schema_version': [1, 0, 1],
- 'source_name': 'hello',
- 'source_copyright': [{
- 'file': 'report.spdx',
- 'sha256': spdx_report_sha256
- }, {
- 'file': 'LICENSES/CC0-1.0.txt',
- 'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt']
- }],
- 'type': 'resource',
- 'identifier': 'hello-message',
- 'long_name': 'Hello Message',
- 'uuid': '1ec36229-298c-4b35-8105-c4f2e1b9811e',
- 'version': [2021, 11, 10],
- 'revision': 2,
- 'description': 'define messages for saying hello and bye',
- 'dependencies': [],
- 'scripts': [{
- 'file': 'message.js',
- 'sha256': file_sha256_hashes['message.js']
- }]
- }]
-
- assert set([rj['identifier'] for rj in expected_resource_jsons]) == \
+ assert set([rj['identifier'] for rj in settings.expected_resources]) == \
set([path.name for path in resource_dir.iterdir()])
- for resource_json in expected_resource_jsons:
+ for resource_json in settings.expected_resources:
subdir = resource_dir / resource_json['identifier']
assert ['2021.11.10'] == [path.name for path in subdir.iterdir()]
@@ -144,93 +330,28 @@ def test_build(tmpdir):
subdir = mapping_dir / 'helloapple'
assert ['2021.11.10'] == [path.name for path in subdir.iterdir()]
- expected_mapping_json = {
- 'api_schema_version': [1, 0, 1],
- 'source_name': 'hello',
- 'source_copyright': [{
- 'file': 'report.spdx',
- 'sha256': spdx_report_sha256
- }, {
- 'file': 'LICENSES/CC0-1.0.txt',
- 'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt']
- }],
- 'type': 'mapping',
- 'identifier': 'helloapple',
- 'long_name': 'Hello Apple',
- 'uuid': '54d23bba-472e-42f5-9194-eaa24c0e3ee7',
- 'version': [2021, 11, 10],
- 'description': 'causes apple to get greeted on Hydrillabugs issue tracker',
- 'payloads': {
- 'https://hydrillabugs.koszko.org/***': {
- 'identifier': 'helloapple'
- },
- 'https://hachettebugs.koszko.org/***': {
- 'identifier': 'helloapple'
- }
- }
- }
-
with open(subdir / '2021.11.10', 'rt') as file_handle:
- assert json.load(file_handle) == expected_mapping_json
+ assert json.load(file_handle) == settings.expected_mapping
# Verify files under 'source/'
source_dir = dstdir / 'source'
assert {'hello.json', 'hello.zip'} == \
set([path.name for path in source_dir.iterdir()])
- src_filenames = (
- *dist_filenames,
- 'README.txt', 'README.txt.license', '.reuse/dep5', 'index.json'
- )
- zip_filenames = [f'hello/{fn}' for fn in src_filenames]
+ zip_filenames = [f'hello/{fn}' for fn in settings.src_filenames]
with ZipFile(source_dir / 'hello.zip', 'r') as archive:
assert set([f.filename for f in archive.filelist]) == set(zip_filenames)
- for zip_fn, src_fn in zip(zip_filenames, src_filenames):
- src_path = srcdir
- for segment in src_fn.split('/'):
- src_path = src_path / segment
-
+ for zip_fn, src_fn in zip(zip_filenames, settings.src_filenames):
with archive.open(zip_fn, 'r') as zip_file_handle:
- with open(src_path, 'rb') as disk_file_handle:
- assert zip_file_handle.read() == disk_file_handle.read()
+ assert zip_file_handle.read() == settings.contents[src_fn]
+ zip_ref = settings.expected_source_description['source_archives']['zip']
with open(source_dir / 'hello.zip', 'rb') as file_handle:
- zipfile_sha256_hash = sha256(file_handle.read()).digest().hex()
-
- expected_source_description = {
- 'api_schema_version': [1, 0, 1],
- 'source_name': 'hello',
- 'source_copyright': [{
- 'file': 'report.spdx',
- 'sha256': spdx_report_sha256
- }, {
- 'file': 'LICENSES/CC0-1.0.txt',
- 'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt']
- }],
- 'source_archives': {
- 'zip': {
- 'sha256': zipfile_sha256_hash,
- }
- },
- 'upstream_url': 'https://git.koszko.org/hydrilla-source-package-example',
- 'definitions': [{
- 'type': 'resource',
- 'identifier': 'helloapple',
- 'version': [2021, 11, 10],
- }, {
- 'type': 'resource',
- 'identifier': 'hello-message',
- 'version': [2021, 11, 10],
- }, {
- 'type': 'mapping',
- 'identifier': 'helloapple',
- 'version': [2021, 11, 10],
- }]
- }
+ zip_ref['sha256'] = sha256(file_handle.read()).digest().hex()
with open(source_dir / 'hello.json', 'rt') as file_handle:
- assert json.load(file_handle) == expected_source_description
+ assert json.load(file_handle) == settings.expected_source_description
-# TODO: also verify on slightly different examples and check error handling
+# TODO: also check error handling
generated by cgit