diff options
| -rw-r--r-- | src/hydrilla_builder/build.py | 14 | ||||
| m--------- | src/hydrilla_builder/schemas | 0 | ||||
| -rw-r--r-- | src/test/test_hydrilla_builder.py | 419 |
3 files changed, 279 insertions, 154 deletions
diff --git a/src/hydrilla_builder/build.py b/src/hydrilla_builder/build.py index dcd7e01..e7c6962 100644 --- a/src/hydrilla_builder/build.py +++ b/src/hydrilla_builder/build.py @@ -223,8 +223,8 @@ class Build: index_obj = json.loads(strip_json_comments(index_json_text)) - self.files_by_path[self.index_json_path] = \ - FileRef(self.index_json_path, index_json_text.encode()) + self.files_by_path[self.srcdir / 'index.json'] = \ + FileRef(self.srcdir / 'index.json', index_json_text.encode()) self._process_index_json(index_obj) @@ -302,7 +302,7 @@ class Build: """ copy_props = ['type', 'identifier', 'long_name', 'uuid', 'description'] if 'comment' in item_def: - copy_props.append(copy_props) + copy_props.append('comment') if item_def['type'] == 'resource': item_list = self.resource_list @@ -319,8 +319,9 @@ class Build: else: item_list = self.mapping_list - payloads = dict([(pat, {'identifier': res_ref['identifier']}) - for pat, res_ref in item_def['payloads'].items()]) + payloads = {} + for pat, res_ref in item_def.get('payloads', {}).items(): + payloads[pat] = {'identifier': res_ref['identifier']} new_item_obj = { 'payloads': payloads @@ -386,6 +387,9 @@ class Build: 'source_archives': source_archives_obj } + if 'comment' in index_obj: + self.source_description['comment'] = index_obj['comment'] + def write_source_package_zip(self, dstpath: Path): """ Create a .zip archive containing files needed to build this source diff --git a/src/hydrilla_builder/schemas b/src/hydrilla_builder/schemas -Subproject 7ce7ee1a4821b23909c07491798083c4b3639f9 +Subproject ca1de2ed4a69a71f2f75552ade693d04ea1baa8 diff --git a/src/test/test_hydrilla_builder.py b/src/test/test_hydrilla_builder.py index 52ed650..b2f19f0 100644 --- a/src/test/test_hydrilla_builder.py +++ b/src/test/test_hydrilla_builder.py @@ -14,20 +14,257 @@ from zipfile import ZipFile here = Path(__file__).resolve().parent +default_srcdir = here / 'source-package-example' + +default_js_filenames = ['bye.js', 'hello.js', 'message.js'] +default_dist_filenames = [*default_js_filenames, 'LICENSES/CC0-1.0.txt'] +default_src_filenames = [ + *default_dist_filenames, + 'README.txt', 'README.txt.license', '.reuse/dep5', 'index.json' +] + +default_sha1_hashes = {} +default_sha256_hashes = {} +default_contents = {} + +for fn in default_src_filenames: + with open(default_srcdir / fn, 'rb') as file_handle: + default_contents[fn] = file_handle.read() + default_sha256_hashes[fn] = sha256(default_contents[fn]).digest().hex() + default_sha1_hashes[fn] = sha1(default_contents[fn]).digest().hex() + +class CaseSettings: + """Gather parametrized values in a class.""" + def __init__(self): + """Init CaseSettings with default values.""" + self.srcdir = default_srcdir + self.index_json_path = Path('index.json') + self.report_spdx_included = True + + self.js_filenames = default_js_filenames.copy() + self.dist_filenames = default_dist_filenames.copy() + self.src_filenames = default_src_filenames.copy() + + self.sha1_hashes = default_sha1_hashes.copy() + self.sha256_hashes = default_sha256_hashes.copy() + self.contents = default_contents.copy() + + self.expected_resources = [{ + 'api_schema_version': [1, 0, 1], + 'source_name': 'hello', + 'source_copyright': [{ + 'file': 'report.spdx', + 'sha256': '!!!!value to fill during test!!!!' + }, { + 'file': 'LICENSES/CC0-1.0.txt', + 'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt'] + }], + 'type': 'resource', + 'identifier': 'helloapple', + 'long_name': 'Hello Apple', + 'uuid': 'a6754dcb-58d8-4b7a-a245-24fd7ad4cd68', + 'version': [2021, 11, 10], + 'revision': 1, + 'description': 'greets an apple', + 'dependencies': ['hello-message'], + 'scripts': [{ + 'file': 'hello.js', + 'sha256': self.sha256_hashes['hello.js'] + }, { + 'file': 'bye.js', + 'sha256': self.sha256_hashes['bye.js'] + }] + }, { + 'api_schema_version': [1, 0, 1], + 'source_name': 'hello', + 'source_copyright': [{ + 'file': 'report.spdx', + 'sha256': '!!!!value to fill during test!!!!' + }, { + 'file': 'LICENSES/CC0-1.0.txt', + 'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt'] + }], + 'type': 'resource', + 'identifier': 'hello-message', + 'long_name': 'Hello Message', + 'uuid': '1ec36229-298c-4b35-8105-c4f2e1b9811e', + 'version': [2021, 11, 10], + 'revision': 2, + 'description': 'define messages for saying hello and bye', + 'dependencies': [], + 'scripts': [{ + 'file': 'message.js', + 'sha256': self.sha256_hashes['message.js'] + }] + }] + self.expected_mapping = { + 'api_schema_version': [1, 0, 1], + 'source_name': 'hello', + 'source_copyright': [{ + 'file': 'report.spdx', + 'sha256': '!!!!value to fill during test!!!!' + }, { + 'file': 'LICENSES/CC0-1.0.txt', + 'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt'] + }], + 'type': 'mapping', + 'identifier': 'helloapple', + 'long_name': 'Hello Apple', + 'uuid': '54d23bba-472e-42f5-9194-eaa24c0e3ee7', + 'version': [2021, 11, 10], + 'description': 'causes apple to get greeted on Hydrillabugs issue tracker', + 'payloads': { + 'https://hydrillabugs.koszko.org/***': { + 'identifier': 'helloapple' + }, + 'https://hachettebugs.koszko.org/***': { + 'identifier': 'helloapple' + } + } + } + self.expected_source_description = { + 'api_schema_version': [1, 0, 1], + 'source_name': 'hello', + 'source_copyright': [{ + 'file': 'report.spdx', + 'sha256': '!!!!value to fill during test!!!!' + }, { + 'file': 'LICENSES/CC0-1.0.txt', + 'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt'] + }], + 'source_archives': { + 'zip': { + 'sha256': '!!!!value to fill during test!!!!', + } + }, + 'upstream_url': 'https://git.koszko.org/hydrilla-source-package-example', + 'definitions': [{ + 'type': 'resource', + 'identifier': 'helloapple', + 'version': [2021, 11, 10], + }, { + 'type': 'resource', + 'identifier': 'hello-message', + 'version': [2021, 11, 10], + }, { + 'type': 'mapping', + 'identifier': 'helloapple', + 'version': [2021, 11, 10], + }] + } + + def expected(self) -> list[dict]: + """ + Convenience method to get a list of expected jsons of 2 resources, + 1 mapping and 1 source description we have. + """ + return [ + *self.expected_resources, + self.expected_mapping, + self.expected_source_description + ] + @pytest.fixture() -def tmpdir(): +def tmpdir() -> str: with TemporaryDirectory() as tmpdir: yield tmpdir -def test_build(tmpdir): +def prepare_default(tmpdir: Path) -> CaseSettings: + """Use sample source package directory as exists in VCS.""" + return CaseSettings() + +def prepare_external_index_json(tmpdir: Path) -> dict: + """ + Use sample source package directory with an alternative, modified + index.json. + """ + settings = CaseSettings() + + from hydrilla_builder.build import strip_json_comments + + with open(settings.srcdir / 'index.json', 'rt') as file_handle: + obj = json.loads(strip_json_comments(file_handle.read())) + + # Add comments that should be preserved. + for dictionary in (obj, settings.expected_source_description): + dictionary['comment'] = 'index_json comment' + + for i, dicts in enumerate(zip(obj['definitions'], settings.expected())): + for dictionary in dicts: + dictionary['comment'] = f'item {i}' + + # Remove spdx report generation + del obj['reuse_generate_spdx_report'] + obj['copyright'].remove({'file': 'report.spdx'}) + + settings.report_spdx_included = False + + for json_description in settings.expected(): + json_description['source_copyright'] = \ + [fr for fr in json_description['source_copyright'] + if fr['file'] != 'report.spdx'] + + # Use default value ([]) for 'additionall_files' property + del obj['additional_files'] + + settings.src_filenames = [*settings.dist_filenames, 'index.json'] + + # Use default value ([]) for 'scripts' property in one of the resources + del obj['definitions'][1]['scripts'] + + settings.expected_resources[1]['scripts'] = [] + + for prefix in ('js', 'dist', 'src'): + getattr(settings, f'{prefix}_filenames').remove('message.js') + + # Use default value ({}) for 'pyloads' property in mapping + del obj['definitions'][2]['payloads'] + + settings.expected_mapping['payloads'] = {} + + # Add some unrecognized properties that should be stripped + to_process = [obj] + while to_process: + processed = to_process.pop() + + if type(processed) is list: + to_process.extend(processed) + elif type(processed) is dict and 'spurious_property' not in processed: + to_process.extend(processed.values()) + processed['spurious_property'] = 'some value' + + # Replace the other index.json with new one + settings.index_json_path = tmpdir / 'replacement.json' + + contents = json.dumps(obj).encode() + + settings.contents['index.json'] = contents + + settings.sha256_hashes['index.json'] = sha256(contents).digest().hex() + settings.sha1_hashes['index.json'] = sha1(contents).digest().hex() + + with open(settings.index_json_path, 'wb') as file_handle: + file_handle.write(contents) + + return settings + +@pytest.mark.parametrize('prepare_source_example', [ + prepare_default, prepare_external_index_json +]) +def test_build(tmpdir, prepare_source_example): """Build the sample source package and verify the produced files.""" from hydrilla_builder.build import Build # First, build the package - srcdir = here / 'source-package-example' - dstdir = Path(tmpdir) + dstdir = Path(tmpdir) / 'dstdir' + tmpdir = Path(tmpdir) / 'example' + + dstdir.mkdir(exist_ok=True) + tmpdir.mkdir(exist_ok=True) - build = Build(srcdir, Path('index.json')) + settings = prepare_source_example(tmpdir) + + build = Build(settings.srcdir, settings.index_json_path) build.write_package_files(dstdir) # Verify directories under destination directory @@ -36,35 +273,25 @@ def test_build(tmpdir): # Verify files under 'file/' file_dir = dstdir / 'file' - js_filenames = ('bye.js', 'hello.js', 'message.js') - dist_filenames = (*js_filenames, str(Path('LICENSES') / 'CC0-1.0.txt')) - file_contents = {} - file_sha256_hashes = {} - file_sha1_hashes = {} - - for fn in dist_filenames: - with open(srcdir / fn, 'rb') as file_handle: - file_contents = file_handle.read() - file_sha256_hashes[fn] = sha256(file_contents).digest().hex() - file_sha1_hashes[fn] = sha1(file_contents).digest().hex() - - dist_file_path = file_dir / f'sha256-{file_sha256_hashes[fn]}' + for fn in settings.dist_filenames: + dist_file_path = file_dir / f'sha256-{settings.sha256_hashes[fn]}' assert dist_file_path.is_file() with open(dist_file_path, 'rb') as file_handle: - assert file_handle.read() == file_contents + assert file_handle.read() == settings.contents[fn] + + sha256_hashes_set = set([settings.sha256_hashes[fn] + for fn in settings.dist_filenames]) - sha256_hashes = set(file_sha256_hashes.values()) spdx_report_sha256 = None - spdx_report_checked = False for path in file_dir.iterdir(): assert path.name.startswith('sha256-') - if path.name[7:] in sha256_hashes: + if path.name[7:] in sha256_hashes_set: continue - assert spdx_report_sha256 is None + assert spdx_report_sha256 is None and settings.report_spdx_included with open(path, 'rt') as file_handle: spdx_contents = file_handle.read() @@ -72,65 +299,24 @@ def test_build(tmpdir): spdx_report_sha256 = sha256(spdx_contents.encode()).digest().hex() assert spdx_report_sha256 == path.name[7:] - for fn in js_filenames: - assert file_sha1_hashes[fn] in spdx_contents + for fn in settings.src_filenames: + if not any([n in fn.lower() for n in ('license', 'reuse')]): + assert settings.sha1_hashes[fn] + + if settings.report_spdx_included: + assert spdx_report_sha256 + for obj in settings.expected(): + for file_ref in obj['source_copyright']: + if file_ref['file'] == 'report.spdx': + file_ref['sha256'] = spdx_report_sha256 # Verify files under 'resource/' resource_dir = dstdir / 'resource' - expected_resource_jsons = [{ - 'api_schema_version': [1, 0, 1], - 'source_name': 'hello', - 'source_copyright': [{ - 'file': 'report.spdx', - 'sha256': spdx_report_sha256 - }, { - 'file': 'LICENSES/CC0-1.0.txt', - 'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt'] - }], - 'type': 'resource', - 'identifier': 'helloapple', - 'long_name': 'Hello Apple', - 'uuid': 'a6754dcb-58d8-4b7a-a245-24fd7ad4cd68', - 'version': [2021, 11, 10], - 'revision': 1, - 'description': 'greets an apple', - 'dependencies': ['hello-message'], - 'scripts': [{ - 'file': 'hello.js', - 'sha256': file_sha256_hashes['hello.js'] - }, { - 'file': 'bye.js', - 'sha256': file_sha256_hashes['bye.js'] - }] - }, { - 'api_schema_version': [1, 0, 1], - 'source_name': 'hello', - 'source_copyright': [{ - 'file': 'report.spdx', - 'sha256': spdx_report_sha256 - }, { - 'file': 'LICENSES/CC0-1.0.txt', - 'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt'] - }], - 'type': 'resource', - 'identifier': 'hello-message', - 'long_name': 'Hello Message', - 'uuid': '1ec36229-298c-4b35-8105-c4f2e1b9811e', - 'version': [2021, 11, 10], - 'revision': 2, - 'description': 'define messages for saying hello and bye', - 'dependencies': [], - 'scripts': [{ - 'file': 'message.js', - 'sha256': file_sha256_hashes['message.js'] - }] - }] - - assert set([rj['identifier'] for rj in expected_resource_jsons]) == \ + assert set([rj['identifier'] for rj in settings.expected_resources]) == \ set([path.name for path in resource_dir.iterdir()]) - for resource_json in expected_resource_jsons: + for resource_json in settings.expected_resources: subdir = resource_dir / resource_json['identifier'] assert ['2021.11.10'] == [path.name for path in subdir.iterdir()] @@ -144,93 +330,28 @@ def test_build(tmpdir): subdir = mapping_dir / 'helloapple' assert ['2021.11.10'] == [path.name for path in subdir.iterdir()] - expected_mapping_json = { - 'api_schema_version': [1, 0, 1], - 'source_name': 'hello', - 'source_copyright': [{ - 'file': 'report.spdx', - 'sha256': spdx_report_sha256 - }, { - 'file': 'LICENSES/CC0-1.0.txt', - 'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt'] - }], - 'type': 'mapping', - 'identifier': 'helloapple', - 'long_name': 'Hello Apple', - 'uuid': '54d23bba-472e-42f5-9194-eaa24c0e3ee7', - 'version': [2021, 11, 10], - 'description': 'causes apple to get greeted on Hydrillabugs issue tracker', - 'payloads': { - 'https://hydrillabugs.koszko.org/***': { - 'identifier': 'helloapple' - }, - 'https://hachettebugs.koszko.org/***': { - 'identifier': 'helloapple' - } - } - } - with open(subdir / '2021.11.10', 'rt') as file_handle: - assert json.load(file_handle) == expected_mapping_json + assert json.load(file_handle) == settings.expected_mapping # Verify files under 'source/' source_dir = dstdir / 'source' assert {'hello.json', 'hello.zip'} == \ set([path.name for path in source_dir.iterdir()]) - src_filenames = ( - *dist_filenames, - 'README.txt', 'README.txt.license', '.reuse/dep5', 'index.json' - ) - zip_filenames = [f'hello/{fn}' for fn in src_filenames] + zip_filenames = [f'hello/{fn}' for fn in settings.src_filenames] with ZipFile(source_dir / 'hello.zip', 'r') as archive: assert set([f.filename for f in archive.filelist]) == set(zip_filenames) - for zip_fn, src_fn in zip(zip_filenames, src_filenames): - src_path = srcdir - for segment in src_fn.split('/'): - src_path = src_path / segment - + for zip_fn, src_fn in zip(zip_filenames, settings.src_filenames): with archive.open(zip_fn, 'r') as zip_file_handle: - with open(src_path, 'rb') as disk_file_handle: - assert zip_file_handle.read() == disk_file_handle.read() + assert zip_file_handle.read() == settings.contents[src_fn] + zip_ref = settings.expected_source_description['source_archives']['zip'] with open(source_dir / 'hello.zip', 'rb') as file_handle: - zipfile_sha256_hash = sha256(file_handle.read()).digest().hex() - - expected_source_description = { - 'api_schema_version': [1, 0, 1], - 'source_name': 'hello', - 'source_copyright': [{ - 'file': 'report.spdx', - 'sha256': spdx_report_sha256 - }, { - 'file': 'LICENSES/CC0-1.0.txt', - 'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt'] - }], - 'source_archives': { - 'zip': { - 'sha256': zipfile_sha256_hash, - } - }, - 'upstream_url': 'https://git.koszko.org/hydrilla-source-package-example', - 'definitions': [{ - 'type': 'resource', - 'identifier': 'helloapple', - 'version': [2021, 11, 10], - }, { - 'type': 'resource', - 'identifier': 'hello-message', - 'version': [2021, 11, 10], - }, { - 'type': 'mapping', - 'identifier': 'helloapple', - 'version': [2021, 11, 10], - }] - } + zip_ref['sha256'] = sha256(file_handle.read()).digest().hex() with open(source_dir / 'hello.json', 'rt') as file_handle: - assert json.load(file_handle) == expected_source_description + assert json.load(file_handle) == settings.expected_source_description -# TODO: also verify on slightly different examples and check error handling +# TODO: also check error handling |