From 367ea85057368047a50ae98a3510e0113eadd744 Mon Sep 17 00:00:00 2001 From: Wojtek Kosior Date: Thu, 25 Aug 2022 16:37:53 +0200 Subject: [proxy] make it possible to uninstall a package This commit also brings some more refactoring under state_impl/. --- .../proxy/state_impl/_operations/__init__.py | 2 +- .../proxy/state_impl/_operations/load_packages.py | 13 +- .../proxy/state_impl/_operations/prune_orphans.py | 145 +++++++++++++++++++++ .../proxy/state_impl/_operations/prune_packages.py | 145 --------------------- .../_operations/recompute_dependencies.py | 58 +++++++-- src/hydrilla/proxy/state_impl/base.py | 28 +++- src/hydrilla/proxy/state_impl/concrete_state.py | 16 ++- src/hydrilla/proxy/state_impl/mappings.py | 82 +++++++----- src/hydrilla/proxy/state_impl/repos.py | 26 +--- 9 files changed, 288 insertions(+), 227 deletions(-) create mode 100644 src/hydrilla/proxy/state_impl/_operations/prune_orphans.py delete mode 100644 src/hydrilla/proxy/state_impl/_operations/prune_packages.py (limited to 'src/hydrilla/proxy/state_impl') diff --git a/src/hydrilla/proxy/state_impl/_operations/__init__.py b/src/hydrilla/proxy/state_impl/_operations/__init__.py index ff34b0b..359e2f5 100644 --- a/src/hydrilla/proxy/state_impl/_operations/__init__.py +++ b/src/hydrilla/proxy/state_impl/_operations/__init__.py @@ -4,7 +4,7 @@ # # Available under the terms of Creative Commons Zero v1.0 Universal. -from .prune_packages import prune_packages +from .prune_orphans import prune_orphans from .pull_missing_files import pull_missing_files from .load_packages import _load_packages_no_state_update from .recompute_dependencies import _recompute_dependencies_no_state_update diff --git a/src/hydrilla/proxy/state_impl/_operations/load_packages.py b/src/hydrilla/proxy/state_impl/_operations/load_packages.py index 107e8d6..f8fddfa 100644 --- a/src/hydrilla/proxy/state_impl/_operations/load_packages.py +++ b/src/hydrilla/proxy/state_impl/_operations/load_packages.py @@ -45,7 +45,7 @@ from .... import item_infos from ... import state from .recompute_dependencies import _recompute_dependencies_no_state_update, \ FileResolver -from .prune_packages import prune_packages +from .prune_orphans import prune_orphans def make_repo_iteration(cursor: sqlite3.Cursor, repo_id: int) -> int: cursor.execute( @@ -401,11 +401,16 @@ def _load_packages_no_state_update( repo_id = repo_id ) - prune_packages(cursor) + if repo_id != 1: + # In case of local semirepo (repo_id = 1) all packages from previous + # iteration are already orphans and can be assumed to be in a pruned + # state no matter what. + prune_orphans(cursor) _recompute_dependencies_no_state_update( - cursor = cursor, - semirepo_file_resolver = MalcontentFileResolver(malcontent_path) + cursor = cursor, + unlocked_required_mappings = [], + semirepo_file_resolver = MalcontentFileResolver(malcontent_path) ) return repo_iteration_id diff --git a/src/hydrilla/proxy/state_impl/_operations/prune_orphans.py b/src/hydrilla/proxy/state_impl/_operations/prune_orphans.py new file mode 100644 index 0000000..f4ebd52 --- /dev/null +++ b/src/hydrilla/proxy/state_impl/_operations/prune_orphans.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +# Haketilo proxy data and configuration (removal of packages that are not used). +# +# This file is part of Hydrilla&Haketilo. +# +# Copyright (C) 2022 Wojtek Kosior +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# I, Wojtek Kosior, thereby promise not to sue for violation of this +# file's license. Although I request that you do not make use this code +# in a proprietary program, I am not going to enforce this in court. + +""" +.... +""" + +# Enable using with Python 3.7. +from __future__ import annotations + +import sqlite3 + +from pathlib import Path + + +_remove_item_versions_sqls = [ + ''' + CREATE TEMPORARY TABLE __removed_versions( + item_version_id INTEGER PRIMARY KEY + ); + ''', ''' + INSERT INTO + __removed_versions + SELECT + iv.item_version_id + FROM + item_versions AS iv + JOIN orphan_iterations AS oi USING (repo_iteration_id) + WHERE + iv.installed != 'I'; + ''', ''' + UPDATE + mapping_statuses + SET + active_version_id = NULL + WHERE + active_version_id IN __removed_versions; + ''', ''' + DELETE FROM + item_versions + WHERE + item_version_id IN __removed_versions; + ''', ''' + DROP TABLE __removed_versions; + ''' +] + +_remove_items_sql = ''' +WITH removed_items AS ( + SELECT + i.item_id + FROM + items AS i + LEFT JOIN item_versions AS iv USING (item_id) + LEFT JOIN mapping_statuses AS ms USING (item_id) + WHERE + iv.item_version_id IS NULL AND + (i.type = 'R' OR ms.enabled = 'N') +) +DELETE FROM + items +WHERE + item_id IN removed_items; +''' + +_remove_files_sql = ''' +WITH removed_files AS ( + SELECT + f.file_id + FROM + files AS f + LEFT JOIN file_uses AS fu USING (file_id) + WHERE + fu.file_use_id IS NULL +) +DELETE FROM + files +WHERE + file_id IN removed_files; +''' + +_remove_repo_iterations_sql = ''' +WITH removed_iterations AS ( + SELECT + oi.repo_iteration_id + FROM + orphan_iterations AS oi + LEFT JOIN item_versions AS iv USING (repo_iteration_id) + WHERE + iv.item_version_id IS NULL +) +DELETE FROM + repo_iterations +WHERE + repo_iteration_id IN removed_iterations; +''' + +_remove_repos_sql = ''' +WITH removed_repos AS ( + SELECT + r.repo_id + FROM + repos AS r + LEFT JOIN repo_iterations AS ri USING (repo_id) + WHERE + r.deleted AND ri.repo_iteration_id IS NULL AND r.repo_id != 1 +) +DELETE FROM + repos +WHERE + repo_id IN removed_repos; +''' + +def prune_orphans(cursor: sqlite3.Cursor) -> None: + assert cursor.connection.in_transaction + + for sql in _remove_item_versions_sqls: + cursor.execute(sql) + cursor.execute(_remove_items_sql) + cursor.execute(_remove_files_sql) + cursor.execute(_remove_repo_iterations_sql) + cursor.execute(_remove_repos_sql) diff --git a/src/hydrilla/proxy/state_impl/_operations/prune_packages.py b/src/hydrilla/proxy/state_impl/_operations/prune_packages.py deleted file mode 100644 index 6f4b3e7..0000000 --- a/src/hydrilla/proxy/state_impl/_operations/prune_packages.py +++ /dev/null @@ -1,145 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -# Haketilo proxy data and configuration (removal of packages that are not used). -# -# This file is part of Hydrilla&Haketilo. -# -# Copyright (C) 2022 Wojtek Kosior -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# -# I, Wojtek Kosior, thereby promise not to sue for violation of this -# file's license. Although I request that you do not make use this code -# in a proprietary program, I am not going to enforce this in court. - -""" -.... -""" - -# Enable using with Python 3.7. -from __future__ import annotations - -import sqlite3 - -from pathlib import Path - - -_remove_item_versions_sqls = [ - ''' - CREATE TEMPORARY TABLE removed_versions( - item_version_id INTEGER PRIMARY KEY - ); - ''', ''' - INSERT INTO - removed_versions - SELECT - iv.item_version_id - FROM - item_versions AS iv - JOIN orphan_iterations AS oi USING (repo_iteration_id) - WHERE - iv.installed != 'I'; - ''', ''' - UPDATE - mapping_statuses - SET - active_version_id = NULL - WHERE - active_version_id IN removed_versions; - ''', ''' - DELETE FROM - item_versions - WHERE - item_version_id IN removed_versions; - ''', ''' - DROP TABLE removed_versions; - ''' -] - -_remove_items_sql = ''' -WITH removed_items AS ( - SELECT - i.item_id - FROM - items AS i - LEFT JOIN item_versions AS iv USING (item_id) - LEFT JOIN mapping_statuses AS ms USING (item_id) - WHERE - iv.item_version_id IS NULL AND - (i.type = 'R' OR ms.enabled = 'N') -) -DELETE FROM - items -WHERE - item_id IN removed_items; -''' - -_remove_files_sql = ''' -WITH removed_files AS ( - SELECT - f.file_id - FROM - files AS f - LEFT JOIN file_uses AS fu USING (file_id) - WHERE - fu.file_use_id IS NULL -) -DELETE FROM - files -WHERE - file_id IN removed_files; -''' - -_remove_repo_iterations_sql = ''' -WITH removed_iterations AS ( - SELECT - oi.repo_iteration_id - FROM - orphan_iterations AS oi - LEFT JOIN item_versions AS iv USING (repo_iteration_id) - WHERE - iv.item_version_id IS NULL -) -DELETE FROM - repo_iterations -WHERE - repo_iteration_id IN removed_iterations; -''' - -_remove_repos_sql = ''' -WITH removed_repos AS ( - SELECT - r.repo_id - FROM - repos AS r - LEFT JOIN repo_iterations AS ri USING (repo_id) - WHERE - r.deleted AND ri.repo_iteration_id IS NULL AND r.repo_id != 1 -) -DELETE FROM - repos -WHERE - repo_id IN removed_repos; -''' - -def prune_packages(cursor: sqlite3.Cursor) -> None: - assert cursor.connection.in_transaction - - for sql in _remove_item_versions_sqls: - cursor.execute(sql) - cursor.execute(_remove_items_sql) - cursor.execute(_remove_files_sql) - cursor.execute(_remove_repo_iterations_sql) - cursor.execute(_remove_repos_sql) diff --git a/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py b/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py index d4c4d45..327a195 100644 --- a/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py +++ b/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py @@ -36,6 +36,7 @@ import typing as t from .... import item_infos from ... import simple_dependency_satisfying as sds +from .. import base from .pull_missing_files import pull_missing_files, FileResolver, \ DummyFileResolver @@ -73,6 +74,41 @@ def _get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar],) \ return result +def _get_current_required_state( + cursor: sqlite3.Cursor, + unlocked_required_mappings: t.Sequence[int] +) -> list[sds.MappingRequirement]: + # For mappings explicitly enabled by the user (+ all mappings they + # recursively depend on) let's make sure that their exact same versions will + # be enabled after the change. Make exception for mappings specified by the + # caller. + with base.temporary_ids_table( + cursor = cursor, + ids = unlocked_required_mappings, + table_name = '__unlocked_ids' + ): + cursor.execute( + ''' + SELECT + definition, repo, repo_iteration + FROM + item_versions_extra + WHERE + item_id NOT IN __unlocked_ids AND active = 'R'; + ''', + ) + + rows = cursor.fetchall() + + requirements: list[sds.MappingRequirement] = [] + + for definition, repo, iteration in rows: + info = item_infos.MappingInfo.load(definition, repo, iteration) + req = sds.MappingVersionRequirement(info.identifier, info) + requirements.append(req) + + return requirements + def _mark_version_installed(cursor: sqlite3.Cursor, version_id: int) -> None: cursor.execute( ''' @@ -87,8 +123,8 @@ def _mark_version_installed(cursor: sqlite3.Cursor, version_id: int) -> None: ) def _recompute_dependencies_no_state_update_no_pull_files( - cursor: sqlite3.Cursor, - extra_requirements: t.Iterable[sds.MappingRequirement] + cursor: sqlite3.Cursor, + unlocked_required_mappings: base.NoLockArg = [], ) -> None: cursor.execute('DELETE FROM payloads;') @@ -98,7 +134,13 @@ def _recompute_dependencies_no_state_update_no_pull_files( resources_to_ids = dict((info, id) for id, info in ids_to_resources.items()) mappings_to_ids = dict((info, id) for id, info in ids_to_mappings.items()) - requirements = [*extra_requirements] + if unlocked_required_mappings == 'all_mappings_unlocked': + requirements = [] + else: + requirements = _get_current_required_state( + cursor = cursor, + unlocked_required_mappings = unlocked_required_mappings + ) cursor.execute( ''' @@ -276,13 +318,13 @@ def _recompute_dependencies_no_state_update_no_pull_files( def _recompute_dependencies_no_state_update( - cursor: sqlite3.Cursor, - extra_requirements: t.Iterable[sds.MappingRequirement] = (), - semirepo_file_resolver: FileResolver = DummyFileResolver() + cursor: sqlite3.Cursor, + unlocked_required_mappings: base.NoLockArg = [], + semirepo_file_resolver: FileResolver = DummyFileResolver() ) -> None: _recompute_dependencies_no_state_update_no_pull_files( - cursor, - extra_requirements + cursor = cursor, + unlocked_required_mappings = unlocked_required_mappings ) pull_missing_files(cursor, semirepo_file_resolver) diff --git a/src/hydrilla/proxy/state_impl/base.py b/src/hydrilla/proxy/state_impl/base.py index a889e71..e5a9898 100644 --- a/src/hydrilla/proxy/state_impl/base.py +++ b/src/hydrilla/proxy/state_impl/base.py @@ -49,6 +49,25 @@ from .. import state as st from .. import policies +@contextmanager +def temporary_ids_table( + cursor: sqlite3.Cursor, + ids: t.Iterable[int], + table_name: str = '__helper_ids' +) -> t.Iterator[None]: + cursor.execute( + f'CREATE TEMPORARY TABLE "{table_name}"(id INTEGER PRIMARY KEY);' + ) + + try: + for id in ids: + cursor.execute(f'INSERT INTO "{table_name}" VALUES(?);', (id,)) + + yield + finally: + cursor.execute(f'DROP TABLE "{table_name}";') + + @dc.dataclass(frozen=True) class PolicyTree(pattern_tree.PatternTree[policies.PolicyFactory]): SelfType = t.TypeVar('SelfType', bound='PolicyTree') @@ -105,6 +124,7 @@ def mark_failed_file_installs( (file_sha256, repo_id) ) +NoLockArg = t.Union[t.Sequence[int], t.Literal['all_mappings_unlocked']] PayloadsData = t.Mapping[st.PayloadRef, st.PayloadData] @@ -199,13 +219,15 @@ class HaketiloStateWithFields(st.HaketiloState): def import_items(self, malcontent_path: Path, repo_id: int = 1) -> None: ... + @abstractmethod + def prune_orphans(self) -> None: + ... + @abstractmethod def recompute_dependencies( self, - requirements: t.Iterable[sds.MappingRequirement] = [], - prune_orphans: bool = False + unlocked_required_mappings: NoLockArg = [] ) -> None: - """....""" ... @abstractmethod diff --git a/src/hydrilla/proxy/state_impl/concrete_state.py b/src/hydrilla/proxy/state_impl/concrete_state.py index 0de67e0..f180ec6 100644 --- a/src/hydrilla/proxy/state_impl/concrete_state.py +++ b/src/hydrilla/proxy/state_impl/concrete_state.py @@ -140,20 +140,22 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields): self.rebuild_structures() + def prune_orphans(self) -> None: + with self.cursor() as cursor: + assert self.connection.in_transaction + + _operations.prune_orphans(cursor) + def recompute_dependencies( self, - extra_requirements: t.Iterable[sds.MappingRequirement] = [], - prune_orphans: bool = False, + unlocked_required_mappings: base.NoLockArg = [] ) -> None: with self.cursor() as cursor: assert self.connection.in_transaction - if prune_orphans: - _operations.prune_packages(cursor) - _operations._recompute_dependencies_no_state_update( - cursor = cursor, - extra_requirements = extra_requirements + cursor = cursor, + unlocked_required_mappings = unlocked_required_mappings ) self.rebuild_structures() diff --git a/src/hydrilla/proxy/state_impl/mappings.py b/src/hydrilla/proxy/state_impl/mappings.py index 8a401b8..eb8b4d2 100644 --- a/src/hydrilla/proxy/state_impl/mappings.py +++ b/src/hydrilla/proxy/state_impl/mappings.py @@ -183,11 +183,8 @@ class ConcreteMappingStore(st.MappingStore): class ConcreteMappingVersionRef(st.MappingVersionRef): state: base.HaketiloStateWithFields = dc.field(hash=False, compare=False) - def _set_installed_status( - self, - cursor: sqlite3.Cursor, - new_status: st.InstalledStatus - ) -> None: + def _set_installed_status(self, cursor: sqlite3.Cursor, new_status: str) \ + -> None: cursor.execute( ''' UPDATE @@ -197,50 +194,63 @@ class ConcreteMappingVersionRef(st.MappingVersionRef): WHERE item_version_id = ?; ''', - (new_status.value, self.id,) + (new_status, self.id,) ) - def install(self) -> None: - with self.state.cursor(transaction=True) as cursor: - cursor.execute( - ''' - SELECT - installed - FROM - item_versions - WHERE - item_version_id = ?; - ''', - (self.id,) - ) + def _get_statuses(self, cursor: sqlite3.Cursor) -> t.Tuple[str, str]: + cursor.execute( + ''' + SELECT + installed, active + FROM + item_versions + WHERE + item_version_id = ?; + ''', + (self.id,) + ) - rows = cursor.fetchall() + rows = cursor.fetchall() - if rows == []: - raise st.MissingItemError() + if rows == []: + raise st.MissingItemError() + + (installed_status, active_status), = rows - (installed_status,), = rows + return installed_status, active_status + + def install(self) -> None: + with self.state.cursor(transaction=True) as cursor: + installed_status, _ = self._get_statuses(cursor) if installed_status == 'I': return - self._set_installed_status(cursor, st.InstalledStatus.INSTALLED) + self._set_installed_status(cursor, 'I') self.state.pull_missing_files() def uninstall(self) -> None: - raise NotImplementedError() - # with self.state.cursor(transaction=True) as cursor: - # info = self.get_display_info() - - # if info.installed == st.InstalledStatus.NOT_INSTALLED: - # return - - # if info.installed == st.InstalledStatus.FAILED_TO_INSTALL: - # self._set_installed_status(st.InstalledStatus.UNINSTALLED) - # return - # - # .... + with self.state.cursor(transaction=True) as cursor: + installed_status, active_status = self._get_statuses(cursor) + + if installed_status == 'N': + return + + self._set_installed_status(cursor, 'N') + + self.state.prune_orphans() + + if active_status == 'R': + self.state.recompute_dependencies() + + cursor.execute( + 'SELECT COUNT(*) FROM item_versions WHERE item_version_id = ?;', + (self.id,) + ) + + (version_still_present,), = cursor.fetchall() + return self if version_still_present else None def get_all_version_display_infos(self) \ -> t.Sequence[st.MappingVersionDisplayInfo]: diff --git a/src/hydrilla/proxy/state_impl/repos.py b/src/hydrilla/proxy/state_impl/repos.py index 838698c..8a3fe64 100644 --- a/src/hydrilla/proxy/state_impl/repos.py +++ b/src/hydrilla/proxy/state_impl/repos.py @@ -193,28 +193,8 @@ class ConcreteRepoRef(st.RepoRef): (self.id,) ) - # For mappings explicitly enabled by the user (+ all mappings they - # recursively depend on) let's make sure that their exact same - # versions will be enabled after the change. - cursor.execute( - ''' - SELECT - definition, repo, repo_iteration - FROM - item_versions_extra - WHERE - active = 'R'; - ''' - ) - - requirements = [] - - for definition, repo, iteration in cursor.fetchall(): - info = item_infos.MappingInfo.load(definition, repo, iteration) - req = sds.MappingVersionRequirement(info.identifier, info) - requirements.append(req) - - self.state.recompute_dependencies(requirements, prune_orphans=True) + self.state.prune_orphans() + self.state.recompute_dependencies() def update( self, @@ -255,7 +235,7 @@ class ConcreteRepoRef(st.RepoRef): except sqlite3.IntegrityError: raise st.RepoNameTaken() - self.state.recompute_dependencies() + self.state.rebuild_structures() def refresh(self) -> None: with self.state.cursor(transaction=True) as cursor: -- cgit v1.2.3