summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/hydrilla/proxy/state.py5
-rw-r--r--src/hydrilla/proxy/state_impl/_operations/load_packages.py17
-rw-r--r--src/hydrilla/proxy/state_impl/_operations/prune_packages.py62
-rw-r--r--src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py78
-rw-r--r--src/hydrilla/proxy/state_impl/concrete_state.py2
-rw-r--r--src/hydrilla/proxy/tables.sql18
6 files changed, 135 insertions, 47 deletions
diff --git a/src/hydrilla/proxy/state.py b/src/hydrilla/proxy/state.py
index 42ca998..d975d2f 100644
--- a/src/hydrilla/proxy/state.py
+++ b/src/hydrilla/proxy/state.py
@@ -94,6 +94,11 @@ class RepoUrlInvalid(HaketiloException):
class RepoCommunicationError(HaketiloException):
pass
+@dc.dataclass
+class FileInstallationError(RepoCommunicationError):
+ repo_id: str
+ sha256: str
+
class RepoApiVersionUnsupported(HaketiloException):
pass
diff --git a/src/hydrilla/proxy/state_impl/_operations/load_packages.py b/src/hydrilla/proxy/state_impl/_operations/load_packages.py
index c294ef0..78e8024 100644
--- a/src/hydrilla/proxy/state_impl/_operations/load_packages.py
+++ b/src/hydrilla/proxy/state_impl/_operations/load_packages.py
@@ -125,7 +125,7 @@ def get_or_make_item(cursor: sqlite3.Cursor, type: str, identifier: str) -> int:
return item_id
-def get_or_make_item_version(
+def make_item_version(
cursor: sqlite3.Cursor,
item_id: int,
repo_iteration_id: int,
@@ -136,13 +136,14 @@ def get_or_make_item_version(
cursor.execute(
'''
- INSERT OR IGNORE INTO item_versions(
+ INSERT INTO item_versions(
item_id,
version,
+ installed,
repo_iteration_id,
definition
)
- VALUES(?, ?, ?, ?);
+ VALUES(?, ?, 'I', ?, ?);
''',
(item_id, ver_str, repo_iteration_id, definition)
)
@@ -241,7 +242,10 @@ def _add_item(
) -> None:
item_id = get_or_make_item(cursor, info.type_name, info.identifier)
- item_version_id = get_or_make_item_version(
+ if isinstance(info, item_infos.MappingInfo):
+ make_mapping_status(cursor, item_id)
+
+ item_version_id = make_item_version(
cursor,
item_id,
repo_iteration_id,
@@ -249,9 +253,6 @@ def _add_item(
definition
)
- if isinstance(info, item_infos.MappingInfo):
- make_mapping_status(cursor, item_id)
-
file_infos = {}
file_specifiers = [*info.source_copyright]
@@ -352,6 +353,8 @@ def load_packages(
repo_id: int,
package_file_resolver: t.Optional[FileResolver] = None
) -> int:
+ assert cursor.connection.in_transaction
+
if package_file_resolver is None:
package_file_resolver = MalcontentFileResolver(malcontent_path)
diff --git a/src/hydrilla/proxy/state_impl/_operations/prune_packages.py b/src/hydrilla/proxy/state_impl/_operations/prune_packages.py
index 9c2b1d7..eb0539c 100644
--- a/src/hydrilla/proxy/state_impl/_operations/prune_packages.py
+++ b/src/hydrilla/proxy/state_impl/_operations/prune_packages.py
@@ -36,61 +36,38 @@ import sqlite3
from pathlib import Path
-_remove_mapping_versions_sqls = [
+_remove_item_versions_sqls = [
'''
- CREATE TEMPORARY TABLE removed_mappings(
+ CREATE TEMPORARY TABLE removed_versions(
item_version_id INTEGER PRIMARY KEY
);
''', '''
INSERT INTO
- removed_mappings
+ removed_versions
SELECT
iv.item_version_id
FROM
item_versions AS iv
- JOIN items AS i USING (item_id)
- JOIN mapping_statuses AS ms USING (item_id)
JOIN orphan_iterations AS oi USING (repo_iteration_id)
WHERE
- NOT ms.required;
+ iv.installed != 'I';
''', '''
UPDATE
mapping_statuses
SET
active_version_id = NULL
WHERE
- active_version_id IN removed_mappings;
+ active_version_id IN removed_versions;
''', '''
DELETE FROM
item_versions
WHERE
- item_version_id IN removed_mappings;
+ item_version_id IN removed_versions;
''', '''
- DROP TABLE removed_mappings;
+ DROP TABLE removed_versions;
'''
]
-_remove_resource_versions_sql = '''
-WITH removed_resources AS (
- SELECT
- iv.item_version_id
- FROM
- item_versions AS iv
- JOIN items AS i
- USING (item_id)
- JOIN orphan_iterations AS oi
- USING (repo_iteration_id)
- LEFT JOIN resolved_depended_resources AS rdr
- ON rdr.resource_item_id = iv.item_version_id
- WHERE
- rdr.payload_id IS NULL
-)
-DELETE FROM
- item_versions
-WHERE
- item_version_id IN removed_resources;
-'''
-
_remove_items_sql = '''
WITH removed_items AS (
SELECT
@@ -101,7 +78,7 @@ WITH removed_items AS (
LEFT JOIN mapping_statuses AS ms USING (item_id)
WHERE
iv.item_version_id IS NULL AND
- i.type = 'R' OR ms.enabled = 'N'
+ (i.type = 'R' OR ms.enabled = 'N')
)
DELETE FROM
items
@@ -160,9 +137,28 @@ WHERE
def prune_packages(cursor: sqlite3.Cursor) -> None:
assert cursor.connection.in_transaction
- for sql in _remove_mapping_versions_sqls:
+ print('VERSIONS TO DELETE', cursor.execute('''
+SELECT
+ iv.item_version_id
+ FROM
+ item_versions AS iv
+ JOIN orphan_iterations AS oi USING (repo_iteration_id)
+ WHERE
+ iv.installed != 'I';
+''').fetchall())
+ for sql in _remove_item_versions_sqls:
cursor.execute(sql)
- cursor.execute(_remove_resource_versions_sql)
+ print('ITEMS TO DELETE', cursor.execute('''
+SELECT
+ i.item_id
+ FROM
+ items AS i
+ LEFT JOIN item_versions AS iv USING (item_id)
+ LEFT JOIN mapping_statuses AS ms USING (item_id)
+ WHERE
+ iv.item_version_id IS NULL AND
+ (i.type = 'R' OR ms.enabled = 'N');
+''').fetchall())
cursor.execute(_remove_items_sql)
cursor.execute(_remove_files_sql)
cursor.execute(_remove_repo_iterations_sql)
diff --git a/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py b/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py
index 4093f12..f83eb09 100644
--- a/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py
+++ b/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py
@@ -37,6 +37,7 @@ import sqlite3
from .... import item_infos
from ... import simple_dependency_satisfying as sds
+from ... import state
AnyInfoVar = t.TypeVar(
@@ -45,7 +46,7 @@ AnyInfoVar = t.TypeVar(
item_infos.MappingInfo
)
-def get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar],) \
+def _get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar],) \
-> t.Mapping[int, AnyInfoVar]:
cursor.execute(
'''
@@ -70,14 +71,27 @@ def get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar],) \
return result
-def _recompute_dependencies_no_state_update(
+def _mark_version_installed(cursor: sqlite3.Cursor, version_id: int) -> None:
+ cursor.execute(
+ '''
+ UPDATE
+ item_versions
+ SET
+ installed = 'I'
+ WHERE
+ item_version_id = ?;
+ ''',
+ (version_id,)
+ )
+
+def _recompute_dependencies_no_state_update_no_pull_files(
cursor: sqlite3.Cursor,
extra_requirements: t.Iterable[sds.MappingRequirement]
) -> None:
cursor.execute('DELETE FROM payloads;')
- ids_to_resources = get_infos_of_type(cursor, item_infos.ResourceInfo)
- ids_to_mappings = get_infos_of_type(cursor, item_infos.MappingInfo)
+ ids_to_resources = _get_infos_of_type(cursor, item_infos.ResourceInfo)
+ ids_to_mappings = _get_infos_of_type(cursor, item_infos.MappingInfo)
resources = ids_to_resources.items()
resources_to_ids = dict((info.identifier, id) for id, info in resources)
@@ -148,6 +162,8 @@ def _recompute_dependencies_no_state_update(
for choice in mapping_choices.values():
mapping_ver_id = mappings_to_ids[choice.info.identifier]
+ _mark_version_installed(cursor, mapping_ver_id)
+
cursor.execute(
'''
SELECT
@@ -210,6 +226,9 @@ def _recompute_dependencies_no_state_update(
for res_num, resource_info in enumerate(payload.resources):
resource_ver_id = resources_to_ids[resource_info.identifier]
+
+ _mark_version_installed(cursor, resource_ver_id)
+
cursor.execute(
'''
INSERT INTO resolved_depended_resources(
@@ -221,3 +240,54 @@ def _recompute_dependencies_no_state_update(
''',
(payload_id, resource_ver_id, res_num)
)
+
+def _pull_missing_files(cursor: sqlite3.Cursor) -> None:
+ cursor.execute(
+ '''
+ SELECT DISTINCT
+ f.file_id, f.sha256,
+ r.repo_id, r.url
+ FROM
+ repos AS R
+ JOIN repo_iterations AS ri USING (repo_id)
+ JOIN item_versions AS iv USING (repo_iteration_id)
+ JOIN file_uses AS fu USING (item_version_id)
+ JOIN files AS f USING (file_id)
+ WHERE
+ iv.installed = 'I' AND f.data IS NULL;
+ '''
+ )
+
+ rows = cursor.fetchall()
+
+ for file_id, sha56, repo_id, repo_url in rows:
+ try:
+ response = requests.get(urljoin(repo_url, f'file/sha256/{sha256}'))
+ assert response.ok
+ except:
+ raise state.FileInstallationError(
+ repo_id = str(repo_id),
+ sha256 = sha256
+ )
+
+ cursor.execute(
+ '''
+ UPDATE
+ files
+ SET
+ data = ?
+ WHERE
+ file_id = ?;
+ ''',
+ (response.content, file_id)
+ )
+
+def _recompute_dependencies_no_state_update(
+ cursor: sqlite3.Cursor,
+ extra_requirements: t.Iterable[sds.MappingRequirement]
+) -> None:
+ _recompute_dependencies_no_state_update_no_pull_files(
+ cursor,
+ extra_requirements
+ )
+ _pull_missing_files(cursor)
diff --git a/src/hydrilla/proxy/state_impl/concrete_state.py b/src/hydrilla/proxy/state_impl/concrete_state.py
index 6a59a75..fb4e2ca 100644
--- a/src/hydrilla/proxy/state_impl/concrete_state.py
+++ b/src/hydrilla/proxy/state_impl/concrete_state.py
@@ -123,8 +123,6 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields):
def import_packages(self, malcontent_path: Path) -> None:
with self.cursor(transaction=True) as cursor:
_operations.load_packages(cursor, malcontent_path, 1)
- raise NotImplementedError()
- _operations.prune_packages(cursor)
self.recompute_dependencies()
diff --git a/src/hydrilla/proxy/tables.sql b/src/hydrilla/proxy/tables.sql
index 0c604fc..3b3506d 100644
--- a/src/hydrilla/proxy/tables.sql
+++ b/src/hydrilla/proxy/tables.sql
@@ -167,9 +167,12 @@ CREATE TABLE mapping_statuses(
FOREIGN KEY (item_id)
REFERENCES items (item_id)
ON DELETE CASCADE,
+ -- We'd like to set "active_version_id" to NULL when referenced entry is
+ -- deleted, but we cannot do it with ON DELETE clause because the
+ -- foreign key is composite. For now - this will be done by the
+ -- application.
FOREIGN KEY (active_version_id, item_id)
REFERENCES item_versions (item_version_id, item_id)
- ON DELETE SET NULL
);
CREATE TABLE item_versions(
@@ -177,12 +180,19 @@ CREATE TABLE item_versions(
item_id INTEGER NOT NULL,
version VARCHAR NOT NULL,
+ -- "installed" determines whether item is INSTALLED, is NOT_INSTALLED or
+ -- it FAILED_TO_INSTALL when last tried. If "required" in a row of
+ -- "mapping_statuses is set to TRUE, the mapping version and all
+ -- resource versions corresponding to it are supposed to have
+ -- "installed" set to 'I'.
+ installed CHAR(1) NOT NULL,
repo_iteration_id INTEGER NOT NULL,
definition BLOB NOT NULL,
UNIQUE (item_id, version, repo_iteration_id),
-- Constraint below needed to allow foreign key from "mapping_statuses".
UNIQUE (item_version_id, item_id),
+ CHECK (installed in ('I', 'N', 'F')),
FOREIGN KEY (item_id)
REFERENCES items (item_id),
@@ -281,7 +291,10 @@ CREATE TABLE resolved_depended_resources(
CREATE TABLE files(
file_id INTEGER PRIMARY KEY,
+ -- File's hash as hexadecimal string.
sha256 CHAR(64) NOT NULL,
+ -- The value of "data" - if not NULL - shall be a bytes sequence that
+ -- corresponds the hash stored in "sha256".
data BLOB NULL,
UNIQUE (sha256)
@@ -290,6 +303,9 @@ CREATE TABLE files(
CREATE TABLE file_uses(
file_use_id INTEGER PRIMARY KEY,
+ -- If item version referenced by "item_version_id" has "installed" set
+ -- to 'I', the file referenced by "file_id" is supposed to have "data"
+ -- set to a valid, non-NULL value.
item_version_id INTEGER NOT NULL,
file_id INTEGER NOT NULL,
name VARCHAR NOT NULL,
generated by cgit