# SPDX-License-Identifier: GPL-3.0-or-later # Haketilo proxy data and configuration (update of dependency tree in the db). # # This file is part of Hydrilla&Haketilo. # # Copyright (C) 2022 Wojtek Kosior # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # # I, Wojtek Kosior, thereby promise not to sue for violation of this # file's license. Although I request that you do not make use this code # in a proprietary program, I am not going to enforce this in court. """ .... """ # Enable using with Python 3.7. from __future__ import annotations import sqlite3 import typing as t from urllib.parse import urlparse, urljoin import requests from .... import item_infos from ... import simple_dependency_satisfying as sds from ... import state AnyInfoVar = t.TypeVar( 'AnyInfoVar', item_infos.ResourceInfo, item_infos.MappingInfo ) def _get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar],) \ -> t.Mapping[int, AnyInfoVar]: cursor.execute( ''' SELECT i.item_id, iv.definition, r.name, ri.iteration FROM item_versions AS iv JOIN items AS i USING (item_id) JOIN repo_iterations AS ri USING (repo_iteration_id) JOIN repos AS r USING (repo_id) WHERE i.type = ?; ''', (info_type.type_name[0].upper(),) ) result: dict[int, AnyInfoVar] = {} for item_id, definition, repo_name, repo_iteration in cursor.fetchall(): info = info_type.load(definition, repo_name, repo_iteration) result[item_id] = info return result def _mark_version_installed(cursor: sqlite3.Cursor, version_id: int) -> None: cursor.execute( ''' UPDATE item_versions SET installed = 'I' WHERE item_version_id = ?; ''', (version_id,) ) def _recompute_dependencies_no_state_update_no_pull_files( cursor: sqlite3.Cursor, extra_requirements: t.Iterable[sds.MappingRequirement] ) -> None: cursor.execute('DELETE FROM payloads;') ids_to_resources = _get_infos_of_type(cursor, item_infos.ResourceInfo) ids_to_mappings = _get_infos_of_type(cursor, item_infos.MappingInfo) resources = ids_to_resources.items() resources_to_ids = dict((info.identifier, id) for id, info in resources) mappings = ids_to_mappings.items() mappings_to_ids = dict((info.identifier, id) for id, info in mappings) requirements = [*extra_requirements] cursor.execute( ''' SELECT i.identifier FROM mapping_statuses AS ms JOIN items AS i USING(item_id) WHERE ms.enabled = 'E' AND ms.frozen = 'N'; ''' ) for mapping_identifier, in cursor.fetchall(): requirements.append(sds.MappingRequirement(mapping_identifier)) cursor.execute( ''' SELECT active_version_id, frozen FROM mapping_statuses WHERE enabled = 'E' AND frozen IN ('R', 'E'); ''' ) for active_version_id, frozen in cursor.fetchall(): info = ids_to_mappings[active_version_id] requirement: sds.MappingRequirement if frozen == 'R': requirement = sds.MappingRepoRequirement(info.identifier, info.repo) else: requirement = sds.MappingVersionRequirement(info.identifier, info) requirements.append(requirement) mapping_choices = sds.compute_payloads( ids_to_resources.values(), ids_to_mappings.values(), requirements ) cursor.execute( ''' UPDATE mapping_statuses SET required = FALSE, active_version_id = NULL WHERE enabled != 'E'; ''' ) cursor.execute('DELETE FROM payloads;') for choice in mapping_choices.values(): mapping_ver_id = mappings_to_ids[choice.info.identifier] _mark_version_installed(cursor, mapping_ver_id) cursor.execute( ''' SELECT item_id FROM item_versions WHERE item_version_id = ?; ''', (mapping_ver_id,) ) (mapping_item_id,), = cursor.fetchall() cursor.execute( ''' UPDATE mapping_statuses SET required = ?, active_version_id = ? WHERE item_id = ?; ''', (choice.required, mapping_ver_id, mapping_item_id) ) for num, (pattern, payload) in enumerate(choice.payloads.items()): cursor.execute( ''' INSERT INTO payloads( mapping_item_id, pattern, eval_allowed, cors_bypass_allowed ) VALUES (?, ?, ?, ?); ''', ( mapping_ver_id, pattern.orig_url, payload.allows_eval, payload.allows_cors_bypass ) ) cursor.execute( ''' SELECT payload_id FROM payloads WHERE mapping_item_id = ? AND pattern = ?; ''', (mapping_ver_id, pattern.orig_url) ) (payload_id,), = cursor.fetchall() for res_num, resource_info in enumerate(payload.resources): resource_ver_id = resources_to_ids[resource_info.identifier] _mark_version_installed(cursor, resource_ver_id) cursor.execute( ''' INSERT INTO resolved_depended_resources( payload_id, resource_item_id, idx ) VALUES(?, ?, ?); ''', (payload_id, resource_ver_id, res_num) ) def _pull_missing_files(cursor: sqlite3.Cursor) -> None: cursor.execute( ''' SELECT DISTINCT f.file_id, f.sha256, r.repo_id, r.url FROM repos AS R JOIN repo_iterations AS ri USING (repo_id) JOIN item_versions AS iv USING (repo_iteration_id) JOIN file_uses AS fu USING (item_version_id) JOIN files AS f USING (file_id) WHERE iv.installed = 'I' AND f.data IS NULL; ''' ) rows = cursor.fetchall() for file_id, sha256, repo_id, repo_url in rows: try: response = requests.get(urljoin(repo_url, f'file/sha256/{sha256}')) assert response.ok except: raise state.FileInstallationError( repo_id = str(repo_id), sha256 = sha256 ) cursor.execute( ''' UPDATE files SET data = ? WHERE file_id = ?; ''', (response.content, file_id) ) def _recompute_dependencies_no_state_update( cursor: sqlite3.Cursor, extra_requirements: t.Iterable[sds.MappingRequirement] ) -> None: _recompute_dependencies_no_state_update_no_pull_files( cursor, extra_requirements ) _pull_missing_files(cursor)