summaryrefslogtreecommitdiff
path: root/src/hydrilla
diff options
context:
space:
mode:
authorWojtek Kosior <koszko@koszko.org>2022-08-31 09:25:40 +0200
committerWojtek Kosior <koszko@koszko.org>2022-09-28 12:54:55 +0200
commitf0044a21ea7bbabb633057804e83df884196012b (patch)
tree77a40d526d6f4daa6d2cc065f2ddcae13822d87f /src/hydrilla
parentedd753e9c4e9f885c1a8c5d981f23e115c20f8e1 (diff)
downloadhaketilo-hydrilla-f0044a21ea7bbabb633057804e83df884196012b.tar.gz
haketilo-hydrilla-f0044a21ea7bbabb633057804e83df884196012b.zip
[proxy] make sure that dependency tree recomputation by default activates the same resources that were marked as required before
Diffstat (limited to 'src/hydrilla')
-rw-r--r--src/hydrilla/item_infos.py2
-rw-r--r--src/hydrilla/proxy/simple_dependency_satisfying.py218
-rw-r--r--src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py74
3 files changed, 188 insertions, 106 deletions
diff --git a/src/hydrilla/item_infos.py b/src/hydrilla/item_infos.py
index 525c6e3..3a25f6c 100644
--- a/src/hydrilla/item_infos.py
+++ b/src/hydrilla/item_infos.py
@@ -612,7 +612,7 @@ class MultirepoItemInfo(
"""
assert not self.is_empty()
- return self.get_all(reverse_versions=True)[-1]
+ return self.get_all(reverse_repos=True)[-1]
def options(self, reverse: bool = False) -> t.Sequence[tuple[str, int]]:
return sorted(
diff --git a/src/hydrilla/proxy/simple_dependency_satisfying.py b/src/hydrilla/proxy/simple_dependency_satisfying.py
index 78a1197..a5431f9 100644
--- a/src/hydrilla/proxy/simple_dependency_satisfying.py
+++ b/src/hydrilla/proxy/simple_dependency_satisfying.py
@@ -37,6 +37,9 @@ from __future__ import annotations
import dataclasses as dc
import typing as t
+import functools as ft
+
+from immutables import Map
from ..exceptions import HaketiloException
from .. import item_infos
@@ -73,8 +76,19 @@ class MappingVersionRequirement(MappingRequirement):
return info == self.version_info
+@dc.dataclass(frozen=True)
+class ResourceVersionRequirement:
+ mapping_identifier: str
+ version_info: item_infos.ResourceInfo
+
+ def is_fulfilled_by(self, info: item_infos.ResourceInfo) -> bool:
+ return info == self.version_info
+
+
@dc.dataclass
class ComputedPayload:
+ mapping_identifier: str
+
resources: list[item_infos.ResourceInfo] = dc.field(default_factory=list)
allows_eval: bool = False
@@ -109,38 +123,76 @@ def _mark_mappings(
ComputedChoices = dict[str, MappingChoice]
+def _compute_inter_mapping_deps(choices: ComputedChoices) \
+ -> dict[str, frozenset[str]]:
+ mapping_deps: dict[str, frozenset[str]] = {}
+
+ for mapping_choice in choices.values():
+ specs_to_resolve = [*mapping_choice.info.required_mappings]
+
+ for computed_payload in mapping_choice.payloads.values():
+ for resource_info in computed_payload.resources:
+ specs_to_resolve.extend(resource_info.required_mappings)
+
+ depended = frozenset(spec.identifier for spec in specs_to_resolve)
+ mapping_deps[mapping_choice.info.identifier] = depended
+
+ return mapping_deps
+
@dc.dataclass(frozen=True)
class _ComputationData:
- resources: t.Mapping[str, item_infos.ResourceInfo]
- mappings: t.Mapping[str, item_infos.MappingInfo]
- required: frozenset[str]
+ resources_map: item_infos.MultirepoResourceInfoMap
+ mappings_map: item_infos.MultirepoMappingInfoMap
+
+ mappings_to_reqs: t.Mapping[str, t.Sequence[MappingRequirement]]
+
+ mappings_resources_to_reqs: t.Mapping[
+ tuple[str, str],
+ t.Sequence[ResourceVersionRequirement]
+ ]
def _satisfy_payload_resource_rec(
self,
resource_identifier: str,
processed_resources: set[str],
- computed_payload: ComputedPayload
+ computed_payload: ComputedPayload
) -> t.Optional[ComputedPayload]:
if resource_identifier in processed_resources:
# We forbid circular dependencies.
return None
- resource_info = self.resources.get(resource_identifier)
- if resource_info is None:
+ multirepo_info = self.resources_map.get(resource_identifier)
+ if multirepo_info is None:
return None
- if resource_info in computed_payload.resources:
+ key = (computed_payload.mapping_identifier, resource_identifier)
+ resource_reqs = self.mappings_resources_to_reqs.get(key)
+
+ if resource_reqs is None:
+ info = multirepo_info.default_info
+ else:
+ found = False
+ # From newest to oldest version.
+ for info in multirepo_info.get_all(reverse_versions=True):
+ if all(req.is_fulfilled_by(info) for req in resource_reqs):
+ found = True
+ break
+
+ if not found:
+ return None
+
+ if info in computed_payload.resources:
return computed_payload
processed_resources.add(resource_identifier)
- if resource_info.allows_eval:
+ if info.allows_eval:
computed_payload.allows_eval = True
- if resource_info.allows_cors_bypass:
+ if info.allows_cors_bypass:
computed_payload.allows_cors_bypass = True
- for dependency_spec in resource_info.dependencies:
+ for dependency_spec in info.dependencies:
if self._satisfy_payload_resource_rec(
dependency_spec.identifier,
processed_resources,
@@ -150,67 +202,68 @@ class _ComputationData:
processed_resources.remove(resource_identifier)
- computed_payload.resources.append(resource_info)
+ computed_payload.resources.append(info)
return computed_payload
- def _satisfy_payload_resource(self, resource_identifier: str) \
- -> t.Optional[ComputedPayload]:
+ def _satisfy_payload_resource(
+ self,
+ mapping_identifier: str,
+ resource_identifier: str
+ ) -> t.Optional[ComputedPayload]:
return self._satisfy_payload_resource_rec(
resource_identifier,
set(),
- ComputedPayload()
+ ComputedPayload(mapping_identifier)
)
- def _compute_payloads_no_mapping_requirements(self) -> ComputedChoices:
- computed_result: ComputedChoices = ComputedChoices()
+ def _compute_best_choices(self) -> ComputedChoices:
+ choices = ComputedChoices()
- for mapping_info in self.mappings.values():
- mapping_choice = MappingChoice(mapping_info)
+ for multirepo_info in self.mappings_map.values():
+ choice: t.Optional[MappingChoice] = None
+
+ reqs = self.mappings_to_reqs.get(multirepo_info.identifier)
+ if reqs is None:
+ choice = MappingChoice(multirepo_info.default_info)
+ else:
+ # From newest to oldest version.
+ for info in multirepo_info.get_all(reverse_versions=True):
+ if all(req.is_fulfilled_by(info) for req in reqs):
+ choice = MappingChoice(info=info, required=True)
+ break
+
+ if choice is None:
+ continue
failure = False
- for pattern, resource_spec in mapping_info.payloads.items():
+ for pattern, resource_spec in choice.info.payloads.items():
computed_payload = self._satisfy_payload_resource(
- resource_spec.identifier
+ mapping_identifier = choice.info.identifier,
+ resource_identifier = resource_spec.identifier
)
if computed_payload is None:
failure = True
break
- if mapping_info.allows_eval:
+ if choice.info.allows_eval:
computed_payload.allows_eval = True
- if mapping_info.allows_cors_bypass:
+ if choice.info.allows_cors_bypass:
computed_payload.allows_cors_bypass = True
- mapping_choice.payloads[pattern] = computed_payload
+ choice.payloads[pattern] = computed_payload
if not failure:
- computed_result[mapping_info.identifier] = mapping_choice
-
- return computed_result
-
- def _compute_inter_mapping_deps(self, choices: ComputedChoices) \
- -> dict[str, frozenset[str]]:
- mapping_deps: dict[str, frozenset[str]] = {}
-
- for mapping_choice in choices.values():
- specs_to_resolve = [*mapping_choice.info.required_mappings]
+ choices[choice.info.identifier] = choice
- for computed_payload in mapping_choice.payloads.values():
- for resource_info in computed_payload.resources:
- specs_to_resolve.extend(resource_info.required_mappings)
-
- depended = frozenset(spec.identifier for spec in specs_to_resolve)
- mapping_deps[mapping_choice.info.identifier] = depended
-
- return mapping_deps
+ return choices
def compute_payloads(self) -> ComputedChoices:
- choices = self._compute_payloads_no_mapping_requirements()
+ choices = self._compute_best_choices()
- mapping_deps = self._compute_inter_mapping_deps(choices)
+ mapping_deps = _compute_inter_mapping_deps(choices)
reverse_deps: dict[str, set[str]] = {}
@@ -221,12 +274,12 @@ class _ComputationData:
bad_mappings: set[str] = set()
for depended_identifier in reverse_deps.keys():
- if self.mappings.get(depended_identifier) not in choices:
+ if depended_identifier not in choices:
_mark_mappings(depended_identifier, reverse_deps, bad_mappings)
bad_required_mappings: list[str] = []
- for identifier in self.required:
+ for identifier in self.mappings_to_reqs.keys():
if identifier in bad_mappings or identifier not in choices:
bad_required_mappings.append(identifier)
@@ -234,12 +287,11 @@ class _ComputationData:
raise ImpossibleSituation(frozenset(bad_required_mappings))
for identifier in bad_mappings:
- if identifier in self.mappings:
- choices.pop(identifier, None)
+ choices.pop(identifier, None)
required_mappings: set[str] = set()
- for identifier in self.required:
+ for identifier in self.mappings_to_reqs.keys():
_mark_mappings(identifier, mapping_deps, required_mappings)
for identifier in required_mappings:
@@ -247,45 +299,35 @@ class _ComputationData:
return choices
-
-AnyInfoVar = t.TypeVar(
- 'AnyInfoVar',
- item_infos.ResourceInfo,
- item_infos.MappingInfo
-)
-
-def _choose_newest(infos: t.Iterable[AnyInfoVar]) -> dict[str, AnyInfoVar]:
- best_versions: dict[str, AnyInfoVar] = {}
-
- for info in infos:
- other = best_versions.setdefault(info.identifier, info)
-
- if (other.version, other.repo, info.repo_iteration) < \
- (info.version, info.repo, other.repo_iteration):
- best_versions[info.identifier] = info
-
- return best_versions
-
def compute_payloads(
- resources: t.Iterable[item_infos.ResourceInfo],
- mappings: t.Iterable[item_infos.MappingInfo],
- requirements: t.Iterable[MappingRequirement]
+ resources: t.Iterable[item_infos.ResourceInfo],
+ mappings: t.Iterable[item_infos.MappingInfo],
+ mapping_requirements: t.Iterable[MappingRequirement],
+ resource_requirements: t.Iterable[ResourceVersionRequirement]
) -> ComputedChoices:
- reqs_by_identifier = dict((req.identifier, req) for req in requirements)
-
- filtered_mappings = []
-
- for mapping_info in mappings:
- req = reqs_by_identifier.get(mapping_info.identifier)
- if req is not None and not req.is_fulfilled_by(mapping_info):
- continue
-
- filtered_mappings.append(mapping_info)
-
- best_resources = _choose_newest(resources)
- best_mappings = _choose_newest(filtered_mappings)
-
- required = frozenset(reqs_by_identifier.keys())
-
- return _ComputationData(best_resources, best_mappings, required)\
- .compute_payloads()
+ resources_map: item_infos.MultirepoResourceInfoMap = \
+ ft.reduce(item_infos.register_in_multirepo_map, resources, Map())
+ mappings_map: item_infos.MultirepoMappingInfoMap = \
+ ft.reduce(item_infos.register_in_multirepo_map, mappings, Map())
+
+ mappings_to_reqs: dict[str, list[MappingRequirement]] = {}
+ for mapping_req in mapping_requirements:
+ mappings_to_reqs.setdefault(mapping_req.identifier, [])\
+ .append(mapping_req)
+
+ mappings_resources_to_reqs: dict[
+ tuple[str, str],
+ list[ResourceVersionRequirement]
+ ] = {}
+ for resource_req in resource_requirements:
+ info = resource_req.version_info
+ key = (resource_req.mapping_identifier, info.identifier)
+ mappings_resources_to_reqs.setdefault(key, [])\
+ .append(resource_req)
+
+ return _ComputationData(
+ mappings_map = mappings_map,
+ resources_map = resources_map,
+ mappings_to_reqs = mappings_to_reqs,
+ mappings_resources_to_reqs = mappings_resources_to_reqs
+ ).compute_payloads()
diff --git a/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py b/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py
index 2ec3600..5403ec3 100644
--- a/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py
+++ b/src/hydrilla/proxy/state_impl/_operations/recompute_dependencies.py
@@ -78,7 +78,7 @@ def _get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar],) \
def _get_current_required_state(
cursor: sqlite3.Cursor,
unlocked_required_mappings: t.Sequence[int]
-) -> list[sds.MappingRequirement]:
+) -> tuple[list[sds.MappingRequirement], list[sds.ResourceVersionRequirement]]:
# For mappings explicitly enabled by the user (+ all mappings they
# recursively depend on) let's make sure that their exact same versions will
# be enabled after the change. Make exception for mappings specified by the
@@ -88,6 +88,7 @@ def _get_current_required_state(
ids = unlocked_required_mappings,
table_name = '__unlocked_ids'
):
+ # Describe all required mappings using requirement objects.
cursor.execute(
'''
SELECT
@@ -101,14 +102,52 @@ def _get_current_required_state(
rows = cursor.fetchall()
- requirements: list[sds.MappingRequirement] = []
+ mapping_requirements: list[sds.MappingRequirement] = []
- for definition, repo, iteration in rows:
- info = item_infos.MappingInfo.load(definition, repo, iteration)
- req = sds.MappingVersionRequirement(info.identifier, info)
- requirements.append(req)
+ for definition, repo, iteration in rows:
+ mapping_info = \
+ item_infos.MappingInfo.load(definition, repo, iteration)
+ mapping_req = sds.MappingVersionRequirement(
+ identifier = mapping_info.identifier,
+ version_info = mapping_info
+ )
+ mapping_requirements.append(mapping_req)
+
+ # Describe all required resources using requirement objects.
+ cursor.execute(
+ '''
+ SELECT
+ i_m.identifier,
+ ive_r.definition, ive_r.repo, ive_r.repo_iteration
+ FROM
+ resolved_depended_resources AS rdd
+ JOIN item_versions_extra AS ive_r
+ ON rdd.resource_item_id = ive_r.item_version_id
+ JOIN payloads AS p
+ USING (payload_id)
+ JOIN item_versions AS iv_m
+ ON p.mapping_item_id = iv_m.item_version_id
+ JOIN items AS i_m
+ ON iv_m.item_id = i_m.item_id
+ WHERE
+ i_m.item_id NOT IN __unlocked_ids AND iv_m.active = 'R';
+ ''',
+ )
+
+ rows = cursor.fetchall()
+
+ resource_requirements: list[sds.ResourceVersionRequirement] = []
- return requirements
+ for mapping_identifier, definition, repo, iteration in rows:
+ resource_info = \
+ item_infos.ResourceInfo.load(definition, repo, iteration)
+ resource_req = sds.ResourceVersionRequirement(
+ mapping_identifier = mapping_identifier,
+ version_info = resource_info
+ )
+ resource_requirements.append(resource_req)
+
+ return (mapping_requirements, resource_requirements)
def _mark_version_installed(cursor: sqlite3.Cursor, version_id: int) -> None:
cursor.execute(
@@ -135,13 +174,13 @@ def _recompute_dependencies_no_state_update_no_pull_files(
resources_to_ids = dict((info, id) for id, info in ids_to_resources.items())
mappings_to_ids = dict((info, id) for id, info in ids_to_mappings.items())
- if unlocked_required_mappings == 'all_mappings_unlocked':
- requirements = []
- else:
- requirements = _get_current_required_state(
+ if unlocked_required_mappings != 'all_mappings_unlocked':
+ mapping_reqs, resource_reqs = _get_current_required_state(
cursor = cursor,
unlocked_required_mappings = unlocked_required_mappings
)
+ else:
+ mapping_reqs, resource_reqs = [], []
cursor.execute(
'''
@@ -156,7 +195,7 @@ def _recompute_dependencies_no_state_update_no_pull_files(
)
for mapping_identifier, in cursor.fetchall():
- requirements.append(sds.MappingRequirement(mapping_identifier))
+ mapping_reqs.append(sds.MappingRequirement(mapping_identifier))
cursor.execute(
'''
@@ -179,12 +218,13 @@ def _recompute_dependencies_no_state_update_no_pull_files(
else:
requirement = sds.MappingVersionRequirement(info.identifier, info)
- requirements.append(requirement)
+ mapping_reqs.append(requirement)
mapping_choices = sds.compute_payloads(
- ids_to_resources.values(),
- ids_to_mappings.values(),
- requirements
+ resources = ids_to_resources.values(),
+ mappings = ids_to_mappings.values(),
+ mapping_requirements = mapping_reqs,
+ resource_requirements = resource_reqs
)
cursor.execute(
@@ -243,7 +283,7 @@ def _recompute_dependencies_no_state_update_no_pull_files(
WHERE
item_version_id = ?;
''',
- (mapping_ver_id, 'R' if choice.required else 'A')
+ ('R' if choice.required else 'A', mapping_ver_id)
)
for num, (pattern, payload) in enumerate(choice.payloads.items()):