From c100476b0a34f5098efc96bf2487f09b66b4a6c4 Mon Sep 17 00:00:00 2001 From: Wojtek Kosior Date: Mon, 22 Aug 2022 16:02:10 +0200 Subject: re-enable the functionality to load packages from ZIP file --- src/hydrilla/proxy/state.py | 5 ++ .../proxy/state_impl/_operations/load_packages.py | 17 +++-- .../proxy/state_impl/_operations/prune_packages.py | 62 ++++++++--------- .../_operations/recompute_dependencies.py | 78 ++++++++++++++++++++-- src/hydrilla/proxy/state_impl/concrete_state.py | 2 - src/hydrilla/proxy/tables.sql | 18 ++++- 6 files changed, 135 insertions(+), 47 deletions(-) (limited to 'src') diff --git a/src/hydrilla/proxy/state.py b/src/hydrilla/proxy/state.py index 42ca998..d975d2f 100644 --- a/src/hydrilla/proxy/state.py +++ b/src/hydrilla/proxy/state.py @@ -94,6 +94,11 @@ class RepoUrlInvalid(HaketiloException): class RepoCommunicationError(HaketiloException): pass +@dc.dataclass +class FileInstallationError(RepoCommunicationError): + repo_id: str + sha256: str + class RepoApiVersionUnsupported(HaketiloException): pass diff --git a/src/hydrilla/proxy/state_impl/_operations/load_packages.py b/src/hydrilla/proxy/state_impl/_operations/load_packages.py index c294ef0..78e8024 100644 --- a/src/hydrilla/proxy/state_impl/_operations/load_packages.py +++ b/src/hydrilla/proxy/state_impl/_operations/load_packages.py @@ -125,7 +125,7 @@ def get_or_make_item(cursor: sqlite3.Cursor, type: str, identifier: str) -> int: return item_id -def get_or_make_item_version( +def make_item_version( cursor: sqlite3.Cursor, item_id: int, repo_iteration_id: int, @@ -136,13 +136,14 @@ def get_or_make_item_version( cursor.execute( ''' - INSERT OR IGNORE INTO item_versions( + INSERT INTO item_versions( item_id, version, + installed, repo_iteration_id, definition ) - VALUES(?, ?, ?, ?); + VALUES(?, ?, 'I', ?, ?); ''', (item_id, ver_str, repo_iteration_id, definition) ) @@ -241,7 +242,10 @@ def _add_item( ) -> None: item_id = get_or_make_item(cursor, info.type_name, info.identifier) - item_version_id = get_or_make_item_version( + if isinstance(info, item_infos.MappingInfo): + make_mapping_status(cursor, item_id) + + item_version_id = make_item_version( cursor, item_id, repo_iteration_id, @@ -249,9 +253,6 @@ def _add_item( definition ) - if isinstance(info, item_infos.MappingInfo): - make_mapping_status(cursor, item_id) - file_infos = {} file_specifiers = [*info.source_copyright] @@ -352,6 +353,8 @@ def load_packages( repo_id: int, package_file_resolver: t.Optional[FileResolver] = None ) -> int: + assert cursor.connection.in_transaction + if package_file_resolver is None: package_file_resolver = MalcontentFileResolver(malcontent_path) diff --git a/src/hydrilla/proxy/state_impl/_operations/prune_packages.py b/src/hydrilla/proxy/state_impl/_operations/prune_packages.py index 9c2b1d7..eb0539c 100644 --- a/src/hydrilla/proxy/state_impl/_operations/prune_packages.py +++ b/src/hydrilla/proxy/state_impl/_operations/prune_packages.py @@ -36,61 +36,38 @@ import sqlite3 from pathlib import Path -_remove_mapping_versions_sqls = [ +_remove_item_versions_sqls = [ ''' - CREATE TEMPORARY TABLE removed_mappings( + CREATE TEMPORARY TABLE removed_versions( item_version_id INTEGER PRIMARY KEY ); ''', ''' INSERT INTO - removed_mappings + removed_versions SELECT iv.item_version_id FROM item_versions AS iv - JOIN items AS i USING (item_id) - JOIN mapping_statuses AS ms USING (item_id) JOIN orphan_iterations AS oi USING (repo_iteration_id) WHERE - NOT ms.required; + iv.installed != 'I'; ''', ''' UPDATE mapping_statuses SET active_version_id = NULL WHERE - active_version_id IN removed_mappings; + active_version_id IN removed_versions; ''', ''' DELETE FROM item_versions WHERE - item_version_id IN removed_mappings; + item_version_id IN removed_versions; ''', ''' - DROP TABLE removed_mappings; + DROP TABLE removed_versions; ''' ] -_remove_resource_versions_sql = ''' -WITH removed_resources AS ( - SELECT - iv.item_version_id - FROM - item_versions AS iv - JOIN items AS i - USING (item_id) - JOIN orphan_iterations AS oi - USING (repo_iteration_id) - LEFT JOIN resolved_depended_resources AS rdr - ON rdr.resource_item_id = iv.item_version_id - WHERE - rdr.payload_id IS NULL -) -DELETE FROM - item_versions -WHERE - item_version_id IN removed_resources; -''' - _remove_items_sql = ''' WITH removed_items AS ( SELECT @@ -101,7 +78,7 @@ WITH removed_items AS ( LEFT JOIN mapping_statuses AS ms USING (item_id) WHERE iv.item_version_id IS NULL AND - i.type = 'R' OR ms.enabled = 'N' + (i.type = 'R' OR ms.enabled = 'N') ) DELETE FROM items @@ -160,9 +137,28 @@ WHERE def prune_packages(cursor: sqlite3.Cursor) -> None: assert cursor.connection.in_transaction - for sql in _remove_mapping_versions_sqls: + print('VERSIONS TO DELETE', cursor.execute(''' +SELECT + iv.item_version_id + FROM + item_versions AS iv + JOIN orphan_iterations AS oi USING (repo_iteration_id) + WHERE + iv.installed != 'I'; +''').fetchall()) + for sql in _remove_item_versions_sqls: cursor.execute(sql) - cursor.execute(_remove_resource_versions_sql) + print('ITEMS TO DELETE', cursor.execute(''' +SELECT + i.item_id + FROM + items AS i + LEFT JOIN item_versions AS iv USING (item_id) + LEFT JOIN mapping_statuses AS ms USING (item_id) + WHERE + iv.item_version_id IS NULL AND + (i.type = 'R' OR ms.enabled = 'N'); +''').fetchall()) cursor.execute(_remove_items_sql) cursor.execute(_remove_files_sql) cursor.execute(_remove_repo_iterations_sql) diff --git a/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py b/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py index 4093f12..f83eb09 100644 --- a/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py +++ b/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py @@ -37,6 +37,7 @@ import sqlite3 from .... import item_infos from ... import simple_dependency_satisfying as sds +from ... import state AnyInfoVar = t.TypeVar( @@ -45,7 +46,7 @@ AnyInfoVar = t.TypeVar( item_infos.MappingInfo ) -def get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar],) \ +def _get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar],) \ -> t.Mapping[int, AnyInfoVar]: cursor.execute( ''' @@ -70,14 +71,27 @@ def get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar],) \ return result -def _recompute_dependencies_no_state_update( +def _mark_version_installed(cursor: sqlite3.Cursor, version_id: int) -> None: + cursor.execute( + ''' + UPDATE + item_versions + SET + installed = 'I' + WHERE + item_version_id = ?; + ''', + (version_id,) + ) + +def _recompute_dependencies_no_state_update_no_pull_files( cursor: sqlite3.Cursor, extra_requirements: t.Iterable[sds.MappingRequirement] ) -> None: cursor.execute('DELETE FROM payloads;') - ids_to_resources = get_infos_of_type(cursor, item_infos.ResourceInfo) - ids_to_mappings = get_infos_of_type(cursor, item_infos.MappingInfo) + ids_to_resources = _get_infos_of_type(cursor, item_infos.ResourceInfo) + ids_to_mappings = _get_infos_of_type(cursor, item_infos.MappingInfo) resources = ids_to_resources.items() resources_to_ids = dict((info.identifier, id) for id, info in resources) @@ -148,6 +162,8 @@ def _recompute_dependencies_no_state_update( for choice in mapping_choices.values(): mapping_ver_id = mappings_to_ids[choice.info.identifier] + _mark_version_installed(cursor, mapping_ver_id) + cursor.execute( ''' SELECT @@ -210,6 +226,9 @@ def _recompute_dependencies_no_state_update( for res_num, resource_info in enumerate(payload.resources): resource_ver_id = resources_to_ids[resource_info.identifier] + + _mark_version_installed(cursor, resource_ver_id) + cursor.execute( ''' INSERT INTO resolved_depended_resources( @@ -221,3 +240,54 @@ def _recompute_dependencies_no_state_update( ''', (payload_id, resource_ver_id, res_num) ) + +def _pull_missing_files(cursor: sqlite3.Cursor) -> None: + cursor.execute( + ''' + SELECT DISTINCT + f.file_id, f.sha256, + r.repo_id, r.url + FROM + repos AS R + JOIN repo_iterations AS ri USING (repo_id) + JOIN item_versions AS iv USING (repo_iteration_id) + JOIN file_uses AS fu USING (item_version_id) + JOIN files AS f USING (file_id) + WHERE + iv.installed = 'I' AND f.data IS NULL; + ''' + ) + + rows = cursor.fetchall() + + for file_id, sha56, repo_id, repo_url in rows: + try: + response = requests.get(urljoin(repo_url, f'file/sha256/{sha256}')) + assert response.ok + except: + raise state.FileInstallationError( + repo_id = str(repo_id), + sha256 = sha256 + ) + + cursor.execute( + ''' + UPDATE + files + SET + data = ? + WHERE + file_id = ?; + ''', + (response.content, file_id) + ) + +def _recompute_dependencies_no_state_update( + cursor: sqlite3.Cursor, + extra_requirements: t.Iterable[sds.MappingRequirement] +) -> None: + _recompute_dependencies_no_state_update_no_pull_files( + cursor, + extra_requirements + ) + _pull_missing_files(cursor) diff --git a/src/hydrilla/proxy/state_impl/concrete_state.py b/src/hydrilla/proxy/state_impl/concrete_state.py index 6a59a75..fb4e2ca 100644 --- a/src/hydrilla/proxy/state_impl/concrete_state.py +++ b/src/hydrilla/proxy/state_impl/concrete_state.py @@ -123,8 +123,6 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields): def import_packages(self, malcontent_path: Path) -> None: with self.cursor(transaction=True) as cursor: _operations.load_packages(cursor, malcontent_path, 1) - raise NotImplementedError() - _operations.prune_packages(cursor) self.recompute_dependencies() diff --git a/src/hydrilla/proxy/tables.sql b/src/hydrilla/proxy/tables.sql index 0c604fc..3b3506d 100644 --- a/src/hydrilla/proxy/tables.sql +++ b/src/hydrilla/proxy/tables.sql @@ -167,9 +167,12 @@ CREATE TABLE mapping_statuses( FOREIGN KEY (item_id) REFERENCES items (item_id) ON DELETE CASCADE, + -- We'd like to set "active_version_id" to NULL when referenced entry is + -- deleted, but we cannot do it with ON DELETE clause because the + -- foreign key is composite. For now - this will be done by the + -- application. FOREIGN KEY (active_version_id, item_id) REFERENCES item_versions (item_version_id, item_id) - ON DELETE SET NULL ); CREATE TABLE item_versions( @@ -177,12 +180,19 @@ CREATE TABLE item_versions( item_id INTEGER NOT NULL, version VARCHAR NOT NULL, + -- "installed" determines whether item is INSTALLED, is NOT_INSTALLED or + -- it FAILED_TO_INSTALL when last tried. If "required" in a row of + -- "mapping_statuses is set to TRUE, the mapping version and all + -- resource versions corresponding to it are supposed to have + -- "installed" set to 'I'. + installed CHAR(1) NOT NULL, repo_iteration_id INTEGER NOT NULL, definition BLOB NOT NULL, UNIQUE (item_id, version, repo_iteration_id), -- Constraint below needed to allow foreign key from "mapping_statuses". UNIQUE (item_version_id, item_id), + CHECK (installed in ('I', 'N', 'F')), FOREIGN KEY (item_id) REFERENCES items (item_id), @@ -281,7 +291,10 @@ CREATE TABLE resolved_depended_resources( CREATE TABLE files( file_id INTEGER PRIMARY KEY, + -- File's hash as hexadecimal string. sha256 CHAR(64) NOT NULL, + -- The value of "data" - if not NULL - shall be a bytes sequence that + -- corresponds the hash stored in "sha256". data BLOB NULL, UNIQUE (sha256) @@ -290,6 +303,9 @@ CREATE TABLE files( CREATE TABLE file_uses( file_use_id INTEGER PRIMARY KEY, + -- If item version referenced by "item_version_id" has "installed" set + -- to 'I', the file referenced by "file_id" is supposed to have "data" + -- set to a valid, non-NULL value. item_version_id INTEGER NOT NULL, file_id INTEGER NOT NULL, name VARCHAR NOT NULL, -- cgit v1.2.3