add test case with modified index.json from the example

1 files changed, 270 insertions, 149 deletions

diff --git a/src/test/test_hydrilla_builder.py b/src/test/test_hydrilla_builder.py
index 52ed650..b2f19f0 100644
--- a/src/test/test_hydrilla_builder.py
+++ b/src/test/test_hydrilla_builder.py
@@ -14,20 +14,257 @@ from zipfile import ZipFile
 
 here = Path(__file__).resolve().parent
 
+default_srcdir = here / 'source-package-example'
+
+default_js_filenames = ['bye.js', 'hello.js', 'message.js']
+default_dist_filenames = [*default_js_filenames, 'LICENSES/CC0-1.0.txt']
+default_src_filenames = [
+    *default_dist_filenames,
+    'README.txt', 'README.txt.license', '.reuse/dep5', 'index.json'
+]
+
+default_sha1_hashes   = {}
+default_sha256_hashes = {}
+default_contents      = {}
+
+for fn in default_src_filenames:
+    with open(default_srcdir / fn, 'rb') as file_handle:
+        default_contents[fn]      = file_handle.read()
+        default_sha256_hashes[fn] = sha256(default_contents[fn]).digest().hex()
+        default_sha1_hashes[fn]   = sha1(default_contents[fn]).digest().hex()
+
+class CaseSettings:
+    """Gather parametrized values in a class."""
+    def __init__(self):
+        """Init CaseSettings with default values."""
+        self.srcdir = default_srcdir
+        self.index_json_path = Path('index.json')
+        self.report_spdx_included = True
+
+        self.js_filenames   = default_js_filenames.copy()
+        self.dist_filenames = default_dist_filenames.copy()
+        self.src_filenames  = default_src_filenames.copy()
+
+        self.sha1_hashes   = default_sha1_hashes.copy()
+        self.sha256_hashes = default_sha256_hashes.copy()
+        self.contents      = default_contents.copy()
+
+        self.expected_resources = [{
+            'api_schema_version': [1, 0, 1],
+            'source_name': 'hello',
+            'source_copyright': [{
+                'file': 'report.spdx',
+                'sha256': '!!!!value to fill during test!!!!'
+            }, {
+                'file': 'LICENSES/CC0-1.0.txt',
+                'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
+            }],
+            'type': 'resource',
+            'identifier': 'helloapple',
+            'long_name': 'Hello Apple',
+            'uuid': 'a6754dcb-58d8-4b7a-a245-24fd7ad4cd68',
+            'version': [2021, 11, 10],
+            'revision': 1,
+            'description': 'greets an apple',
+            'dependencies': ['hello-message'],
+            'scripts': [{
+                'file': 'hello.js',
+                'sha256': self.sha256_hashes['hello.js']
+            }, {
+                'file': 'bye.js',
+                'sha256': self.sha256_hashes['bye.js']
+            }]
+        }, {
+            'api_schema_version': [1, 0, 1],
+            'source_name': 'hello',
+            'source_copyright': [{
+                'file': 'report.spdx',
+                'sha256': '!!!!value to fill during test!!!!'
+            }, {
+                'file': 'LICENSES/CC0-1.0.txt',
+                'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
+            }],
+            'type': 'resource',
+            'identifier': 'hello-message',
+            'long_name': 'Hello Message',
+            'uuid': '1ec36229-298c-4b35-8105-c4f2e1b9811e',
+            'version': [2021, 11, 10],
+            'revision': 2,
+            'description': 'define messages for saying hello and bye',
+            'dependencies': [],
+            'scripts': [{
+                'file': 'message.js',
+                'sha256': self.sha256_hashes['message.js']
+            }]
+        }]
+        self.expected_mapping = {
+            'api_schema_version': [1, 0, 1],
+            'source_name': 'hello',
+            'source_copyright': [{
+                'file': 'report.spdx',
+                'sha256': '!!!!value to fill during test!!!!'
+            }, {
+                'file': 'LICENSES/CC0-1.0.txt',
+                'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
+            }],
+            'type': 'mapping',
+	    'identifier': 'helloapple',
+	    'long_name': 'Hello Apple',
+	    'uuid': '54d23bba-472e-42f5-9194-eaa24c0e3ee7',
+	    'version': [2021, 11, 10],
+	    'description': 'causes apple to get greeted on Hydrillabugs issue tracker',
+	    'payloads': {
+	        'https://hydrillabugs.koszko.org/***': {
+		    'identifier': 'helloapple'
+	        },
+	        'https://hachettebugs.koszko.org/***': {
+		    'identifier': 'helloapple'
+                }
+            }
+        }
+        self.expected_source_description = {
+            'api_schema_version': [1, 0, 1],
+            'source_name': 'hello',
+            'source_copyright': [{
+                'file': 'report.spdx',
+                'sha256': '!!!!value to fill during test!!!!'
+            }, {
+                'file': 'LICENSES/CC0-1.0.txt',
+                'sha256': self.sha256_hashes['LICENSES/CC0-1.0.txt']
+            }],
+            'source_archives': {
+                'zip': {
+                    'sha256': '!!!!value to fill during test!!!!',
+                }
+            },
+            'upstream_url': 'https://git.koszko.org/hydrilla-source-package-example',
+            'definitions': [{
+                'type': 'resource',
+                'identifier': 'helloapple',
+                'version': [2021, 11, 10],
+            }, {
+                'type':       'resource',
+                'identifier': 'hello-message',
+                'version':     [2021, 11, 10],
+            }, {
+                'type': 'mapping',
+                'identifier': 'helloapple',
+                'version': [2021, 11, 10],
+            }]
+        }
+
+    def expected(self) -> list[dict]:
+        """
+        Convenience method to get a list of expected jsons of 2 resources,
+        1 mapping and 1 source description we have.
+        """
+        return [
+            *self.expected_resources,
+            self.expected_mapping,
+            self.expected_source_description
+        ]
+
 @pytest.fixture()
-def tmpdir():
+def tmpdir() -> str:
     with TemporaryDirectory() as tmpdir:
         yield tmpdir
 
-def test_build(tmpdir):
+def prepare_default(tmpdir: Path) -> CaseSettings:
+    """Use sample source package directory as exists in VCS."""
+    return CaseSettings()
+
+def prepare_external_index_json(tmpdir: Path) -> dict:
+    """
+    Use sample source package directory with an alternative, modified
+    index.json.
+    """
+    settings = CaseSettings()
+
+    from hydrilla_builder.build import strip_json_comments
+
+    with open(settings.srcdir / 'index.json', 'rt') as file_handle:
+        obj = json.loads(strip_json_comments(file_handle.read()))
+
+    # Add comments that should be preserved.
+    for dictionary in (obj, settings.expected_source_description):
+        dictionary['comment'] = 'index_json comment'
+
+    for i, dicts in enumerate(zip(obj['definitions'], settings.expected())):
+        for dictionary in dicts:
+            dictionary['comment'] = f'item {i}'
+
+    # Remove spdx report generation
+    del obj['reuse_generate_spdx_report']
+    obj['copyright'].remove({'file': 'report.spdx'})
+
+    settings.report_spdx_included = False
+
+    for json_description in settings.expected():
+        json_description['source_copyright'] = \
+            [fr for fr in json_description['source_copyright']
+             if fr['file'] != 'report.spdx']
+
+    # Use default value ([]) for 'additionall_files' property
+    del obj['additional_files']
+
+    settings.src_filenames = [*settings.dist_filenames, 'index.json']
+
+    # Use default value ([]) for 'scripts' property in one of the resources
+    del obj['definitions'][1]['scripts']
+
+    settings.expected_resources[1]['scripts'] = []
+
+    for prefix in ('js', 'dist', 'src'):
+        getattr(settings, f'{prefix}_filenames').remove('message.js')
+
+    # Use default value ({}) for 'pyloads' property in mapping
+    del obj['definitions'][2]['payloads']
+
+    settings.expected_mapping['payloads'] = {}
+
+    # Add some unrecognized properties that should be stripped
+    to_process = [obj]
+    while to_process:
+        processed = to_process.pop()
+
+        if type(processed) is list:
+            to_process.extend(processed)
+        elif type(processed) is dict and 'spurious_property' not in processed:
+            to_process.extend(processed.values())
+            processed['spurious_property'] = 'some value'
+
+    # Replace the other index.json with new one
+    settings.index_json_path = tmpdir / 'replacement.json'
+
+    contents = json.dumps(obj).encode()
+
+    settings.contents['index.json'] = contents
+
+    settings.sha256_hashes['index.json'] = sha256(contents).digest().hex()
+    settings.sha1_hashes['index.json']   = sha1(contents).digest().hex()
+
+    with open(settings.index_json_path, 'wb') as file_handle:
+        file_handle.write(contents)
+
+    return settings
+
+@pytest.mark.parametrize('prepare_source_example', [
+    prepare_default, prepare_external_index_json
+])
+def test_build(tmpdir, prepare_source_example):
     """Build the sample source package and verify the produced files."""
     from hydrilla_builder.build import Build
 
     # First, build the package
-    srcdir = here / 'source-package-example'
-    dstdir = Path(tmpdir)
+    dstdir = Path(tmpdir) / 'dstdir'
+    tmpdir = Path(tmpdir) / 'example'
+
+    dstdir.mkdir(exist_ok=True)
+    tmpdir.mkdir(exist_ok=True)
 
-    build = Build(srcdir, Path('index.json'))
+    settings = prepare_source_example(tmpdir)
+
+    build = Build(settings.srcdir, settings.index_json_path)
     build.write_package_files(dstdir)
 
     # Verify directories under destination directory
@@ -36,35 +273,25 @@ def test_build(tmpdir):
 
     # Verify files under 'file/'
     file_dir = dstdir / 'file'
-    js_filenames = ('bye.js', 'hello.js', 'message.js')
-    dist_filenames = (*js_filenames, str(Path('LICENSES') / 'CC0-1.0.txt'))
-    file_contents = {}
-    file_sha256_hashes = {}
-    file_sha1_hashes = {}
-
-    for fn in dist_filenames:
-        with open(srcdir / fn, 'rb') as file_handle:
-            file_contents = file_handle.read()
 
-        file_sha256_hashes[fn] = sha256(file_contents).digest().hex()
-        file_sha1_hashes[fn]   = sha1(file_contents).digest().hex()
-
-        dist_file_path = file_dir / f'sha256-{file_sha256_hashes[fn]}'
+    for fn in settings.dist_filenames:
+        dist_file_path = file_dir / f'sha256-{settings.sha256_hashes[fn]}'
         assert dist_file_path.is_file()
 
         with open(dist_file_path, 'rb') as file_handle:
-            assert file_handle.read() == file_contents
+            assert file_handle.read() == settings.contents[fn]
+
+    sha256_hashes_set = set([settings.sha256_hashes[fn]
+                             for fn in settings.dist_filenames])
 
-    sha256_hashes = set(file_sha256_hashes.values())
     spdx_report_sha256 = None
-    spdx_report_checked = False
 
     for path in file_dir.iterdir():
         assert path.name.startswith('sha256-')
-        if path.name[7:] in sha256_hashes:
+        if path.name[7:] in sha256_hashes_set:
             continue
 
-        assert spdx_report_sha256 is None
+        assert spdx_report_sha256 is None and settings.report_spdx_included
 
         with open(path, 'rt') as file_handle:
             spdx_contents = file_handle.read()
@@ -72,65 +299,24 @@ def test_build(tmpdir):
         spdx_report_sha256 = sha256(spdx_contents.encode()).digest().hex()
         assert spdx_report_sha256 == path.name[7:]
 
-        for fn in js_filenames:
-            assert file_sha1_hashes[fn] in spdx_contents
+        for fn in settings.src_filenames:
+            if not any([n in fn.lower() for n in ('license', 'reuse')]):
+                assert settings.sha1_hashes[fn]
+
+    if settings.report_spdx_included:
+        assert spdx_report_sha256
+        for obj in settings.expected():
+            for file_ref in obj['source_copyright']:
+                if file_ref['file'] == 'report.spdx':
+                    file_ref['sha256'] = spdx_report_sha256
 
     # Verify files under 'resource/'
     resource_dir = dstdir / 'resource'
 
-    expected_resource_jsons = [{
-        'api_schema_version': [1, 0, 1],
-        'source_name': 'hello',
-        'source_copyright': [{
-            'file': 'report.spdx',
-            'sha256': spdx_report_sha256
-        }, {
-            'file': 'LICENSES/CC0-1.0.txt',
-            'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt']
-        }],
-        'type': 'resource',
-        'identifier': 'helloapple',
-        'long_name': 'Hello Apple',
-        'uuid': 'a6754dcb-58d8-4b7a-a245-24fd7ad4cd68',
-        'version': [2021, 11, 10],
-        'revision': 1,
-        'description': 'greets an apple',
-        'dependencies': ['hello-message'],
-        'scripts': [{
-            'file': 'hello.js',
-            'sha256': file_sha256_hashes['hello.js']
-        }, {
-            'file': 'bye.js',
-            'sha256': file_sha256_hashes['bye.js']
-        }]
-    }, {
-        'api_schema_version': [1, 0, 1],
-        'source_name': 'hello',
-        'source_copyright': [{
-            'file': 'report.spdx',
-            'sha256': spdx_report_sha256
-        }, {
-            'file': 'LICENSES/CC0-1.0.txt',
-            'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt']
-        }],
-        'type': 'resource',
-        'identifier': 'hello-message',
-        'long_name': 'Hello Message',
-        'uuid': '1ec36229-298c-4b35-8105-c4f2e1b9811e',
-        'version': [2021, 11, 10],
-        'revision': 2,
-        'description': 'define messages for saying hello and bye',
-        'dependencies': [],
-        'scripts': [{
-            'file': 'message.js',
-            'sha256': file_sha256_hashes['message.js']
-        }]
-    }]
-
-    assert set([rj['identifier'] for rj in expected_resource_jsons]) == \
+    assert set([rj['identifier'] for rj in settings.expected_resources]) == \
         set([path.name for path in resource_dir.iterdir()])
 
-    for resource_json in expected_resource_jsons:
+    for resource_json in settings.expected_resources:
         subdir = resource_dir / resource_json['identifier']
         assert ['2021.11.10'] == [path.name for path in subdir.iterdir()]
 
@@ -144,93 +330,28 @@ def test_build(tmpdir):
     subdir = mapping_dir / 'helloapple'
     assert ['2021.11.10'] == [path.name for path in subdir.iterdir()]
 
-    expected_mapping_json = {
-        'api_schema_version': [1, 0, 1],
-        'source_name': 'hello',
-        'source_copyright': [{
-            'file': 'report.spdx',
-            'sha256': spdx_report_sha256
-        }, {
-            'file': 'LICENSES/CC0-1.0.txt',
-            'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt']
-        }],
-        'type': 'mapping',
-	'identifier': 'helloapple',
-	'long_name': 'Hello Apple',
-	'uuid': '54d23bba-472e-42f5-9194-eaa24c0e3ee7',
-	'version': [2021, 11, 10],
-	'description': 'causes apple to get greeted on Hydrillabugs issue tracker',
-	'payloads': {
-	    'https://hydrillabugs.koszko.org/***': {
-		'identifier': 'helloapple'
-	    },
-	    'https://hachettebugs.koszko.org/***': {
-		'identifier': 'helloapple'
-            }
-        }
-    }
-
     with open(subdir / '2021.11.10', 'rt') as file_handle:
-        assert json.load(file_handle) == expected_mapping_json
+        assert json.load(file_handle) == settings.expected_mapping
 
     # Verify files under 'source/'
     source_dir = dstdir / 'source'
     assert {'hello.json', 'hello.zip'} == \
         set([path.name for path in source_dir.iterdir()])
 
-    src_filenames = (
-        *dist_filenames,
-        'README.txt', 'README.txt.license', '.reuse/dep5', 'index.json'
-    )
-    zip_filenames = [f'hello/{fn}' for fn in src_filenames]
+    zip_filenames = [f'hello/{fn}' for fn in settings.src_filenames]
 
     with ZipFile(source_dir / 'hello.zip', 'r') as archive:
         assert set([f.filename for f in archive.filelist]) == set(zip_filenames)
 
-        for zip_fn, src_fn in zip(zip_filenames, src_filenames):
-            src_path = srcdir
-            for segment in src_fn.split('/'):
-                src_path = src_path / segment
-
+        for zip_fn, src_fn in zip(zip_filenames, settings.src_filenames):
             with archive.open(zip_fn, 'r') as zip_file_handle:
-                with open(src_path, 'rb') as disk_file_handle:
-                    assert zip_file_handle.read() == disk_file_handle.read()
+                assert zip_file_handle.read() == settings.contents[src_fn]
 
+    zip_ref = settings.expected_source_description['source_archives']['zip']
     with open(source_dir / 'hello.zip', 'rb') as file_handle:
-        zipfile_sha256_hash = sha256(file_handle.read()).digest().hex()
-
-    expected_source_description = {
-        'api_schema_version': [1, 0, 1],
-        'source_name': 'hello',
-        'source_copyright': [{
-            'file': 'report.spdx',
-            'sha256': spdx_report_sha256
-        }, {
-            'file': 'LICENSES/CC0-1.0.txt',
-            'sha256': file_sha256_hashes['LICENSES/CC0-1.0.txt']
-        }],
-        'source_archives': {
-            'zip': {
-                'sha256': zipfile_sha256_hash,
-            }
-        },
-        'upstream_url': 'https://git.koszko.org/hydrilla-source-package-example',
-        'definitions': [{
-            'type': 'resource',
-            'identifier': 'helloapple',
-            'version': [2021, 11, 10],
-        }, {
-            'type':       'resource',
-            'identifier': 'hello-message',
-            'version':     [2021, 11, 10],
-        }, {
-            'type': 'mapping',
-            'identifier': 'helloapple',
-            'version': [2021, 11, 10],
-        }]
-    }
+        zip_ref['sha256'] = sha256(file_handle.read()).digest().hex()
 
     with open(source_dir / 'hello.json', 'rt') as file_handle:
-        assert json.load(file_handle) == expected_source_description
+        assert json.load(file_handle) == settings.expected_source_description
 
-# TODO: also verify on slightly different examples and check error handling
+# TODO: also check error handling