diff options
Diffstat (limited to 'src/hydrilla/proxy/state_impl/_operations/load_packages.py')
| -rw-r--r-- | src/hydrilla/proxy/state_impl/_operations/load_packages.py | 124 |
1 files changed, 49 insertions, 75 deletions
diff --git a/src/hydrilla/proxy/state_impl/_operations/load_packages.py b/src/hydrilla/proxy/state_impl/_operations/load_packages.py index 78e8024..16d1154 100644 --- a/src/hydrilla/proxy/state_impl/_operations/load_packages.py +++ b/src/hydrilla/proxy/state_impl/_operations/load_packages.py @@ -32,20 +32,19 @@ from __future__ import annotations import io -import hashlib +import mimetypes +import sqlite3 import dataclasses as dc import typing as t -from pathlib import Path -from abc import ABC, abstractmethod - -import sqlite3 +from pathlib import Path, PurePosixPath -from ....exceptions import HaketiloException -from ....translations import smart_gettext as _ from .... import versions from .... import item_infos - +from ... import state +from .recompute_dependencies import _recompute_dependencies_no_state_update, \ + FileResolver +from .prune_packages import prune_packages def make_repo_iteration(cursor: sqlite3.Cursor, repo_id: int) -> int: cursor.execute( @@ -128,8 +127,9 @@ def get_or_make_item(cursor: sqlite3.Cursor, type: str, identifier: str) -> int: def make_item_version( cursor: sqlite3.Cursor, item_id: int, - repo_iteration_id: int, version: versions.VerTuple, + installed: str, + repo_iteration_id: int, definition: bytes ) -> int: ver_str = versions.version_string(version) @@ -143,9 +143,9 @@ def make_item_version( repo_iteration_id, definition ) - VALUES(?, ?, 'I', ?, ?); + VALUES(?, ?, ?, ?, ?); ''', - (item_id, ver_str, repo_iteration_id, definition) + (item_id, ver_str, installed, repo_iteration_id, definition) ) cursor.execute( @@ -173,27 +173,10 @@ def make_mapping_status(cursor: sqlite3.Cursor, item_id: int) -> None: (item_id,) ) -def get_or_make_file(cursor: sqlite3.Cursor, sha256: str, file_bytes: bytes) \ - -> int: - cursor.execute( - ''' - INSERT OR IGNORE INTO files(sha256, data) - VALUES(?, ?) - ''', - (sha256, file_bytes) - ) +def get_or_make_file(cursor: sqlite3.Cursor, sha256: str) -> int: + cursor.execute('INSERT OR IGNORE INTO files(sha256) VALUES(?);', (sha256,)) - cursor.execute( - ''' - SELECT - file_id - FROM - files - WHERE - sha256 = ?; - ''', - (sha256,) - ) + cursor.execute('SELECT file_id FROM files WHERE sha256 = ?;', (sha256,)) (file_id,), = cursor.fetchall() @@ -225,20 +208,15 @@ def make_file_use( @dc.dataclass(frozen=True) class _FileInfo: - id: int - is_ascii: bool - -class FileResolver(ABC): - @abstractmethod - def by_sha256(self, sha256: str) -> bytes: - ... + id: int + extension: str def _add_item( cursor: sqlite3.Cursor, - package_file_resolver: FileResolver, info: item_infos.AnyInfo, definition: bytes, - repo_iteration_id: int + repo_iteration_id: int, + repo_id: int ) -> None: item_id = get_or_make_item(cursor, info.type_name, info.identifier) @@ -246,11 +224,12 @@ def _add_item( make_mapping_status(cursor, item_id) item_version_id = make_item_version( - cursor, - item_id, - repo_iteration_id, - info.version, - definition + cursor = cursor, + item_id = item_id, + version = info.version, + installed = 'I' if repo_id == 1 else 'N', + repo_iteration_id = repo_iteration_id, + definition = definition ) file_infos = {} @@ -260,29 +239,24 @@ def _add_item( file_specifiers.extend(info.scripts) for file_spec in file_specifiers: - file_bytes = package_file_resolver.by_sha256(file_spec.sha256) - - sha256 = hashlib.sha256(file_bytes).digest().hex() - if sha256 != file_spec.sha256: - fmt = _('err.proxy.file_hash_mismatched_{item_identifier}_{file_name}_{expected_sha256}_{actual_sha256}') - msg = fmt.format( - item_identifier = info.identifier, - file_name = file_spec.name, - expected_sha256 = file_spec.sha256, - actual_sha256 = sha256 - ) - raise HaketiloException(msg) + file_id = get_or_make_file(cursor, file_spec.sha256) - file_id = get_or_make_file(cursor, sha256, file_bytes) + suffix = PurePosixPath(file_spec.name).suffix - file_infos[sha256] = _FileInfo(file_id, file_bytes.isascii()) + file_infos[file_spec.sha256] = _FileInfo(file_id, suffix) for idx, file_spec in enumerate(info.source_copyright): file_info = file_infos[file_spec.sha256] - if file_info.is_ascii: - mime = 'text/plain' - else: + + mime = mimetypes.types_map.get(file_info.extension) + if mime is None: + mime = mimetypes.common_types.get(file_info.extension) + if mime is None: mime = 'application/octet-stream' + if mime is None and file_info.extension == '.spdx': + # We don't know of any estabilished mime type for tag-value SPDX + # reports. Let's use the following for now. + mime = 'text/spdx' make_file_use( cursor, @@ -342,22 +316,17 @@ class MalcontentFileResolver(FileResolver): def by_sha256(self, sha256: str) -> bytes: file_path = self.malcontent_dir_path / 'file' / 'sha256' / sha256 if not file_path.is_file(): - fmt = _('err.proxy.file_missing_{sha256}') - raise HaketiloException(fmt.format(sha256=sha256)) + raise state.FileMissingError(repo_id='1', sha256=sha256) return file_path.read_bytes() -def load_packages( +def _load_packages_no_state_update( cursor: sqlite3.Cursor, malcontent_path: Path, - repo_id: int, - package_file_resolver: t.Optional[FileResolver] = None + repo_id: int ) -> int: assert cursor.connection.in_transaction - if package_file_resolver is None: - package_file_resolver = MalcontentFileResolver(malcontent_path) - repo_iteration_id = make_repo_iteration(cursor, repo_id) types: t.Iterable[t.Type[item_infos.AnyInfo]] = \ @@ -371,11 +340,16 @@ def load_packages( info_type ): _add_item( - cursor, - package_file_resolver, - info, - definition, - repo_iteration_id + cursor = cursor, + info = info, + definition = definition, + repo_iteration_id = repo_iteration_id, + repo_id = repo_id ) + _recompute_dependencies_no_state_update( + cursor = cursor, + semirepo_file_resolver = MalcontentFileResolver(malcontent_path) + ) + return repo_iteration_id |