re-enable the functionality to load packages from ZIP file

6 files changed, 135 insertions, 47 deletions

diff --git a/src/hydrilla/proxy/state.py b/src/hydrilla/proxy/state.py
index 42ca998..d975d2f 100644
--- a/src/hydrilla/proxy/state.py
+++ b/src/hydrilla/proxy/state.py
@@ -94,6 +94,11 @@ class RepoUrlInvalid(HaketiloException):
 class RepoCommunicationError(HaketiloException):
     pass
 
+@dc.dataclass
+class FileInstallationError(RepoCommunicationError):
+    repo_id: str
+    sha256:  str
+
 class RepoApiVersionUnsupported(HaketiloException):
     pass
 
diff --git a/src/hydrilla/proxy/state_impl/_operations/load_packages.py b/src/hydrilla/proxy/state_impl/_operations/load_packages.py
index c294ef0..78e8024 100644
--- a/src/hydrilla/proxy/state_impl/_operations/load_packages.py
+++ b/src/hydrilla/proxy/state_impl/_operations/load_packages.py
@@ -125,7 +125,7 @@ def get_or_make_item(cursor: sqlite3.Cursor, type: str, identifier: str) -> int:
 
     return item_id
 
-def get_or_make_item_version(
+def make_item_version(
         cursor:            sqlite3.Cursor,
         item_id:           int,
         repo_iteration_id: int,
@@ -136,13 +136,14 @@ def get_or_make_item_version(
 
     cursor.execute(
         '''
-        INSERT OR IGNORE INTO item_versions(
+        INSERT INTO item_versions(
                 item_id,
                 version,
+                installed,
 	        repo_iteration_id,
                 definition
         )
-        VALUES(?, ?, ?, ?);
+        VALUES(?, ?, 'I', ?, ?);
         ''',
         (item_id, ver_str, repo_iteration_id, definition)
     )
@@ -241,7 +242,10 @@ def _add_item(
 ) -> None:
     item_id = get_or_make_item(cursor, info.type_name, info.identifier)
 
-    item_version_id = get_or_make_item_version(
+    if isinstance(info, item_infos.MappingInfo):
+        make_mapping_status(cursor, item_id)
+
+    item_version_id = make_item_version(
         cursor,
         item_id,
         repo_iteration_id,
@@ -249,9 +253,6 @@ def _add_item(
         definition
     )
 
-    if isinstance(info, item_infos.MappingInfo):
-        make_mapping_status(cursor, item_id)
-
     file_infos = {}
 
     file_specifiers = [*info.source_copyright]
@@ -352,6 +353,8 @@ def load_packages(
         repo_id:               int,
         package_file_resolver: t.Optional[FileResolver] = None
 ) -> int:
+    assert cursor.connection.in_transaction
+
     if package_file_resolver is None:
         package_file_resolver = MalcontentFileResolver(malcontent_path)
 
diff --git a/src/hydrilla/proxy/state_impl/_operations/prune_packages.py b/src/hydrilla/proxy/state_impl/_operations/prune_packages.py
index 9c2b1d7..eb0539c 100644
--- a/src/hydrilla/proxy/state_impl/_operations/prune_packages.py
+++ b/src/hydrilla/proxy/state_impl/_operations/prune_packages.py
@@ -36,61 +36,38 @@ import sqlite3
 from pathlib import Path
 
 
-_remove_mapping_versions_sqls = [
+_remove_item_versions_sqls = [
     '''
-    CREATE TEMPORARY TABLE removed_mappings(
+    CREATE TEMPORARY TABLE removed_versions(
             item_version_id INTEGER PRIMARY KEY
     );
     ''', '''
     INSERT INTO
-            removed_mappings
+            removed_versions
     SELECT
             iv.item_version_id
     FROM
                  item_versions     AS iv
-            JOIN items             AS i  USING (item_id)
-            JOIN mapping_statuses  AS ms USING (item_id)
             JOIN orphan_iterations AS oi USING (repo_iteration_id)
     WHERE
-            NOT ms.required;
+            iv.installed != 'I';
     ''', '''
     UPDATE
             mapping_statuses
     SET
             active_version_id = NULL
     WHERE
-            active_version_id IN removed_mappings;
+            active_version_id IN removed_versions;
     ''', '''
     DELETE FROM
             item_versions
     WHERE
-            item_version_id IN removed_mappings;
+            item_version_id IN removed_versions;
     ''', '''
-    DROP TABLE removed_mappings;
+    DROP TABLE removed_versions;
     '''
 ]
 
-_remove_resource_versions_sql = '''
-WITH removed_resources AS (
-        SELECT
-                iv.item_version_id
-        FROM
-                          item_versions               AS iv
-                JOIN      items                       AS i
-                        USING (item_id)
-                JOIN      orphan_iterations           AS oi
-                        USING (repo_iteration_id)
-                LEFT JOIN resolved_depended_resources AS rdr
-                        ON rdr.resource_item_id = iv.item_version_id
-        WHERE
-                rdr.payload_id IS NULL
-)
-DELETE FROM
-        item_versions
-WHERE
-        item_version_id IN removed_resources;
-'''
-
 _remove_items_sql = '''
 WITH removed_items AS (
         SELECT
@@ -101,7 +78,7 @@ WITH removed_items AS (
                 LEFT JOIN mapping_statuses AS ms USING (item_id)
         WHERE
                  iv.item_version_id IS NULL AND
-                 i.type = 'R' OR ms.enabled = 'N'
+                 (i.type = 'R' OR ms.enabled = 'N')
 )
 DELETE FROM
         items
@@ -160,9 +137,28 @@ WHERE
 def prune_packages(cursor: sqlite3.Cursor) -> None:
     assert cursor.connection.in_transaction
 
-    for sql in _remove_mapping_versions_sqls:
+    print('VERSIONS TO DELETE', cursor.execute('''
+SELECT
+            iv.item_version_id
+    FROM
+                 item_versions     AS iv
+            JOIN orphan_iterations AS oi USING (repo_iteration_id)
+    WHERE
+            iv.installed != 'I';
+''').fetchall())
+    for sql in _remove_item_versions_sqls:
         cursor.execute(sql)
-    cursor.execute(_remove_resource_versions_sql)
+    print('ITEMS TO DELETE', cursor.execute('''
+SELECT
+                i.item_id
+        FROM
+                          items            AS i
+                LEFT JOIN item_versions    AS iv USING (item_id)
+                LEFT JOIN mapping_statuses AS ms USING (item_id)
+        WHERE
+                 iv.item_version_id IS NULL AND
+                 (i.type = 'R' OR ms.enabled = 'N');
+''').fetchall())
     cursor.execute(_remove_items_sql)
     cursor.execute(_remove_files_sql)
     cursor.execute(_remove_repo_iterations_sql)
diff --git a/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py b/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py
index 4093f12..f83eb09 100644
--- a/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py
+++ b/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py
@@ -37,6 +37,7 @@ import sqlite3
 
 from .... import item_infos
 from ... import simple_dependency_satisfying as sds
+from ... import state
 
 
 AnyInfoVar = t.TypeVar(
@@ -45,7 +46,7 @@ AnyInfoVar = t.TypeVar(
     item_infos.MappingInfo
 )
 
-def get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar],) \
+def _get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar],) \
     -> t.Mapping[int, AnyInfoVar]:
     cursor.execute(
         '''
@@ -70,14 +71,27 @@ def get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar],) \
 
     return result
 
-def _recompute_dependencies_no_state_update(
+def _mark_version_installed(cursor: sqlite3.Cursor, version_id: int) -> None:
+    cursor.execute(
+        '''
+        UPDATE
+                item_versions
+        SET
+                installed = 'I'
+        WHERE
+                item_version_id = ?;
+        ''',
+        (version_id,)
+    )
+
+def _recompute_dependencies_no_state_update_no_pull_files(
         cursor:             sqlite3.Cursor,
         extra_requirements: t.Iterable[sds.MappingRequirement]
 ) -> None:
     cursor.execute('DELETE FROM payloads;')
 
-    ids_to_resources = get_infos_of_type(cursor, item_infos.ResourceInfo)
-    ids_to_mappings  = get_infos_of_type(cursor, item_infos.MappingInfo)
+    ids_to_resources = _get_infos_of_type(cursor, item_infos.ResourceInfo)
+    ids_to_mappings  = _get_infos_of_type(cursor, item_infos.MappingInfo)
 
     resources = ids_to_resources.items()
     resources_to_ids = dict((info.identifier, id) for id, info in resources)
@@ -148,6 +162,8 @@ def _recompute_dependencies_no_state_update(
     for choice in mapping_choices.values():
         mapping_ver_id = mappings_to_ids[choice.info.identifier]
 
+        _mark_version_installed(cursor, mapping_ver_id)
+
         cursor.execute(
             '''
             SELECT
@@ -210,6 +226,9 @@ def _recompute_dependencies_no_state_update(
 
             for res_num, resource_info in enumerate(payload.resources):
                 resource_ver_id = resources_to_ids[resource_info.identifier]
+
+                _mark_version_installed(cursor, resource_ver_id)
+
                 cursor.execute(
                     '''
                     INSERT INTO resolved_depended_resources(
@@ -221,3 +240,54 @@ def _recompute_dependencies_no_state_update(
                     ''',
                     (payload_id, resource_ver_id, res_num)
                 )
+
+def _pull_missing_files(cursor: sqlite3.Cursor) -> None:
+    cursor.execute(
+        '''
+        SELECT DISTINCT
+                f.file_id, f.sha256,
+                r.repo_id, r.url
+        FROM
+                     repos           AS R
+                JOIN repo_iterations AS ri USING (repo_id)
+                JOIN item_versions   AS iv USING (repo_iteration_id)
+                JOIN file_uses       AS fu USING (item_version_id)
+                JOIN files           AS f  USING (file_id)
+        WHERE
+                iv.installed = 'I' AND f.data IS NULL;
+        '''
+    )
+
+    rows = cursor.fetchall()
+
+    for file_id, sha56, repo_id, repo_url in rows:
+        try:
+            response = requests.get(urljoin(repo_url, f'file/sha256/{sha256}'))
+            assert response.ok
+        except:
+            raise state.FileInstallationError(
+                repo_id = str(repo_id),
+                sha256  = sha256
+            )
+
+        cursor.execute(
+            '''
+            UPDATE
+                    files
+            SET
+                    data = ?
+            WHERE
+                    file_id = ?;
+            ''',
+            (response.content, file_id)
+        )
+
+def _recompute_dependencies_no_state_update(
+        cursor:             sqlite3.Cursor,
+        extra_requirements: t.Iterable[sds.MappingRequirement]
+) -> None:
+    _recompute_dependencies_no_state_update_no_pull_files(
+        cursor,
+        extra_requirements
+    )
+    _pull_missing_files(cursor)
diff --git a/src/hydrilla/proxy/state_impl/concrete_state.py b/src/hydrilla/proxy/state_impl/concrete_state.py
index 6a59a75..fb4e2ca 100644
--- a/src/hydrilla/proxy/state_impl/concrete_state.py
+++ b/src/hydrilla/proxy/state_impl/concrete_state.py
@@ -123,8 +123,6 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields):
     def import_packages(self, malcontent_path: Path) -> None:
         with self.cursor(transaction=True) as cursor:
             _operations.load_packages(cursor, malcontent_path, 1)
-            raise NotImplementedError()
-            _operations.prune_packages(cursor)
 
             self.recompute_dependencies()
 
diff --git a/src/hydrilla/proxy/tables.sql b/src/hydrilla/proxy/tables.sql
index 0c604fc..3b3506d 100644
--- a/src/hydrilla/proxy/tables.sql
+++ b/src/hydrilla/proxy/tables.sql
@@ -167,9 +167,12 @@ CREATE TABLE mapping_statuses(
 	FOREIGN KEY (item_id)
                 REFERENCES items (item_id)
 		ON DELETE CASCADE,
+        -- We'd like to set "active_version_id" to NULL when referenced entry is
+	-- deleted, but we cannot do it with ON DELETE clause because the
+	-- foreign key is composite. For now - this will be done by the
+	-- application.
 	FOREIGN KEY (active_version_id, item_id)
                 REFERENCES item_versions (item_version_id, item_id)
-		ON DELETE SET NULL
 );
 
 CREATE TABLE item_versions(
@@ -177,12 +180,19 @@ CREATE TABLE item_versions(
 
 	item_id             INTEGER NOT NULL,
         version             VARCHAR NOT NULL,
+	-- "installed" determines whether item is INSTALLED, is NOT_INSTALLED or
+	-- it FAILED_TO_INSTALL when last tried. If "required" in a row of
+	-- "mapping_statuses is set to TRUE, the mapping version and all
+	-- resource versions corresponding to it are supposed to have
+	-- "installed" set to 'I'.
+	installed           CHAR(1) NOT NULL,
 	repo_iteration_id   INTEGER NOT NULL,
 	definition          BLOB    NOT NULL,
 
         UNIQUE (item_id, version, repo_iteration_id),
 	-- Constraint below needed to allow foreign key from "mapping_statuses".
 	UNIQUE (item_version_id, item_id),
+	CHECK (installed in ('I', 'N', 'F')),
 
 	FOREIGN KEY (item_id)
                 REFERENCES items (item_id),
@@ -281,7 +291,10 @@ CREATE TABLE resolved_depended_resources(
 CREATE TABLE files(
         file_id INTEGER PRIMARY KEY,
 
+	-- File's hash as hexadecimal string.
         sha256  CHAR(64) NOT NULL,
+	-- The value of "data" - if not NULL - shall be a bytes sequence that
+	-- corresponds the hash stored in "sha256".
 	data    BLOB     NULL,
 
 	UNIQUE (sha256)
@@ -290,6 +303,9 @@ CREATE TABLE files(
 CREATE TABLE file_uses(
         file_use_id     INTEGER PRIMARY KEY,
 
+	-- If item version referenced by "item_version_id" has "installed" set
+	-- to 'I', the file referenced by "file_id" is supposed to have "data"
+	-- set to a valid, non-NULL value.
 	item_version_id INTEGER NOT NULL,
         file_id         INTEGER NOT NULL,
         name            VARCHAR NOT NULL,