summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorWojtek Kosior <koszko@koszko.org>2022-08-11 09:39:19 +0200
committerWojtek Kosior <koszko@koszko.org>2022-08-11 09:39:19 +0200
commit3f3ba519ae3c3346945928b21ab36f7238e5387e (patch)
tree2852ed975257ee375d185746a7b559d587668aee
parent72fcc76cc75ccb7e180886170db01dae637e250e (diff)
downloadhaketilo-hydrilla-3f3ba519ae3c3346945928b21ab36f7238e5387e.tar.gz
haketilo-hydrilla-3f3ba519ae3c3346945928b21ab36f7238e5387e.zip
save computed payloads into sqlite db
-rw-r--r--src/hydrilla/item_infos.py24
-rw-r--r--src/hydrilla/proxy/state_impl/base.py38
-rw-r--r--src/hydrilla/proxy/state_impl/concrete_state.py113
-rw-r--r--src/hydrilla/proxy/tables.sql20
4 files changed, 120 insertions, 75 deletions
diff --git a/src/hydrilla/item_infos.py b/src/hydrilla/item_infos.py
index c1e1e31..0bdc95e 100644
--- a/src/hydrilla/item_infos.py
+++ b/src/hydrilla/item_infos.py
@@ -153,14 +153,14 @@ class ItemInfoBase(ABC, ItemIdentity, Categorizable):
"""...."""
type_name: t.ClassVar[str]
- source_name: str = dc.field(hash=False)
- source_copyright: tuple[FileSpecifier, ...] = dc.field(hash=False)
- uuid: t.Optional[str] = dc.field(hash=False)
- long_name: str = dc.field(hash=False)
- allows_eval: bool = dc.field(hash=False)
- allows_cors_bypass: bool = dc.field(hash=False)
- required_mappings: tuple[ItemSpecifier, ...] = dc.field(hash=False)
- generated_by: t.Optional[GeneratedBy] = dc.field(hash=False)
+ source_name: str = dc.field(hash=False, compare=False)
+ source_copyright: tuple[FileSpecifier, ...] = dc.field(hash=False, compare=False)
+ uuid: t.Optional[str] = dc.field(hash=False, compare=False)
+ long_name: str = dc.field(hash=False, compare=False)
+ allows_eval: bool = dc.field(hash=False, compare=False)
+ allows_cors_bypass: bool = dc.field(hash=False, compare=False)
+ required_mappings: tuple[ItemSpecifier, ...] = dc.field(hash=False, compare=False)
+ generated_by: t.Optional[GeneratedBy] = dc.field(hash=False, compare=False)
# def path_relative_to_type(self) -> str:
# """
@@ -237,9 +237,9 @@ class ResourceInfo(ItemInfoBase):
"""...."""
type_name: t.ClassVar[str] = 'resource'
- revision: int = dc.field(hash=False)
- dependencies: tuple[ItemSpecifier, ...] = dc.field(hash=False)
- scripts: tuple[FileSpecifier, ...] = dc.field(hash=False)
+ revision: int = dc.field(hash=False, compare=False)
+ dependencies: tuple[ItemSpecifier, ...] = dc.field(hash=False, compare=False)
+ scripts: tuple[FileSpecifier, ...] = dc.field(hash=False, compare=False)
@property
def versioned_identifier(self):
@@ -321,7 +321,7 @@ class MappingInfo(ItemInfoBase):
"""...."""
type_name: t.ClassVar[str] = 'mapping'
- payloads: t.Mapping[ParsedPattern, ItemSpecifier] = dc.field(hash=False)
+ payloads: t.Mapping[ParsedPattern, ItemSpecifier] = dc.field(hash=False, compare=False)
@staticmethod
def make(
diff --git a/src/hydrilla/proxy/state_impl/base.py b/src/hydrilla/proxy/state_impl/base.py
index 78a50c0..788a93d 100644
--- a/src/hydrilla/proxy/state_impl/base.py
+++ b/src/hydrilla/proxy/state_impl/base.py
@@ -72,40 +72,38 @@ class HaketiloStateWithFields(state.HaketiloState):
lock: threading.RLock = dc.field(default_factory=threading.RLock)
@contextmanager
- def cursor(self, lock: bool = False, transaction: bool = False) \
+ def cursor(self, transaction: bool = False) \
-> t.Iterator[sqlite3.Cursor]:
"""...."""
start_transaction = transaction and not self.connection.in_transaction
- assert lock or not start_transaction
-
try:
- if lock:
- self.lock.acquire()
+ self.lock.acquire()
if self.current_cursor is not None:
yield self.current_cursor
return
- self.current_cursor = self.connection.cursor()
-
- if start_transaction:
- self.current_cursor.execute('BEGIN TRANSACTION;')
-
try:
- yield self.current_cursor
+ self.current_cursor = self.connection.cursor()
if start_transaction:
- self.current_cursor.execute('COMMIT TRANSACTION;')
- except:
- if start_transaction:
- self.current_cursor.execute('ROLLBACK TRANSACTION;')
- raise
+ self.current_cursor.execute('BEGIN TRANSACTION;')
+
+ try:
+ yield self.current_cursor
+
+ if start_transaction:
+ assert self.connection.in_transaction
+ self.current_cursor.execute('COMMIT TRANSACTION;')
+ except:
+ if start_transaction:
+ self.current_cursor.execute('ROLLBACK TRANSACTION;')
+ raise
+ finally:
+ self.current_cursor = None
finally:
- self.current_cursor = None
-
- if lock:
- self.lock.release()
+ self.lock.release()
def rebuild_structures(self, cursor: sqlite3.Cursor) -> None:
"""...."""
diff --git a/src/hydrilla/proxy/state_impl/concrete_state.py b/src/hydrilla/proxy/state_impl/concrete_state.py
index 1b46ae9..ccb1269 100644
--- a/src/hydrilla/proxy/state_impl/concrete_state.py
+++ b/src/hydrilla/proxy/state_impl/concrete_state.py
@@ -313,19 +313,10 @@ def get_or_make_item_version(
cursor: sqlite3.Cursor,
item_id: int,
repo_iteration_id: int,
- definition: str,
- info: item_infos.AnyInfo
+ version: versions.VerTuple,
+ definition: str
) -> int:
- ver_str = versions.version_string(info.version)
-
- values = (
- item_id,
- ver_str,
- repo_iteration_id,
- definition,
- info.allows_eval,
- info.allows_cors_bypass
- )
+ ver_str = versions.version_string(version)
cursor.execute(
'''
@@ -333,13 +324,11 @@ def get_or_make_item_version(
item_id,
version,
repo_iteration_id,
- definition,
- eval_allowed,
- cors_bypass_allowed
+ definition
)
- VALUES(?, ?, ?, ?, ?, ?);
+ VALUES(?, ?, ?, ?);
''',
- values
+ (item_id, ver_str, repo_iteration_id, definition)
)
cursor.execute(
@@ -417,12 +406,12 @@ def make_file_use(
(item_version_id, file_id, name, type, mime_type, idx)
)
-def get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar]) \
- -> t.Iterable[AnyInfoVar]:
+def get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar],) \
+ -> t.Mapping[AnyInfoVar, int]:
cursor.execute(
'''
SELECT
- iv.definition, r.name, ri.iteration
+ i.item_id, iv.definition, r.name, ri.iteration
FROM
item_versions AS iv
JOIN items AS i USING (item_id)
@@ -434,11 +423,12 @@ def get_infos_of_type(cursor: sqlite3.Cursor, info_type: t.Type[AnyInfoVar]) \
(info_type.type_name[0].upper(),)
)
- result: list[AnyInfoVar] = []
+ result: dict[AnyInfoVar, int] = {}
- for definition, repo_name, repo_iteration in cursor.fetchall():
+ for item_id, definition, repo_name, repo_iteration in cursor.fetchall():
definition_io = io.StringIO(definition)
- result.append(info_type.load(definition_io, repo_name, repo_iteration))
+ info = info_type.load(definition_io, repo_name, repo_iteration)
+ result[info] = item_id
return result
@@ -449,7 +439,7 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields):
self._populate_database_with_stuff_from_temporary_malcontent_dir()
- with self.cursor() as cursor:
+ with self.cursor(transaction=True) as cursor:
self.rebuild_structures(cursor)
def _prepare_database(self) -> None:
@@ -459,9 +449,12 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields):
try:
cursor.execute(
'''
- SELECT COUNT(name)
- FROM sqlite_master
- WHERE name = 'general' AND type = 'table';
+ SELECT
+ COUNT(name)
+ FROM
+ sqlite_master
+ WHERE
+ name = 'general' AND type = 'table';
'''
)
@@ -473,8 +466,10 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields):
else:
cursor.execute(
'''
- SELECT haketilo_version
- FROM general;
+ SELECT
+ haketilo_version
+ FROM
+ general;
'''
)
@@ -495,7 +490,7 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields):
malcontent_dir_path = self.store_dir / 'temporary_malcontent'
files_by_sha256_path = malcontent_dir_path / 'file' / 'sha256'
- with self.cursor(lock=True, transaction=True) as cursor:
+ with self.cursor(transaction=True) as cursor:
for info_type in [item_infos.ResourceInfo, item_infos.MappingInfo]:
info: item_infos.AnyInfo
for info, definition in read_items(
@@ -517,8 +512,8 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields):
cursor,
item_id,
repo_iteration_id,
- definition,
- info
+ info.version,
+ definition
)
if info_type is item_infos.MappingInfo:
@@ -574,16 +569,66 @@ class ConcreteHaketiloState(base.HaketiloStateWithFields):
)
def rebuild_structures(self, cursor: sqlite3.Cursor) -> None:
+ assert self.connection.in_transaction
+
resources = get_infos_of_type(cursor, item_infos.ResourceInfo)
mappings = get_infos_of_type(cursor, item_infos.MappingInfo)
- payloads = compute_payloads(resources, mappings)
+ payloads = compute_payloads(resources.keys(), mappings.keys())
payloads_data = {}
+ cursor.execute('DELETE FROM payloads;')
+
for mapping_info, by_pattern in payloads.items():
for num, (pattern, payload) in enumerate(by_pattern.items()):
- payload_id = f'{num}@{mapping_info.identifier}'
+ print('adding payload')
+ cursor.execute(
+ '''
+ INSERT INTO payloads(
+ mapping_item_id,
+ pattern,
+ eval_allowed,
+ cors_bypass_allowed
+ )
+ VALUES (?, ?, ?, ?);
+ ''',
+ (
+ mappings[mapping_info],
+ pattern.orig_url,
+ payload.allows_eval,
+ payload.allows_cors_bypass
+ )
+ )
+
+ cursor.execute(
+ '''
+ SELECT
+ payload_id
+ FROM
+ payloads
+ WHERE
+ mapping_item_id = ? AND pattern = ?;
+ ''',
+ (mappings[mapping_info], pattern.orig_url)
+ )
+
+ (payload_id_int,), = cursor.fetchall()
+
+ for res_num, resource_info in enumerate(payload.resources):
+ cursor.execute(
+ '''
+ INSERT INTO resolved_depended_resources(
+ payload_id,
+ resource_item_id,
+ idx
+ )
+ VALUES(?, ?, ?);
+ ''',
+ (payload_id_int, resources[resource_info], res_num)
+ )
+
+ payload_id = str(payload_id_int)
ref = ConcretePayloadRef(payload_id, payload)
diff --git a/src/hydrilla/proxy/tables.sql b/src/hydrilla/proxy/tables.sql
index 53539a7..25493d3 100644
--- a/src/hydrilla/proxy/tables.sql
+++ b/src/hydrilla/proxy/tables.sql
@@ -141,10 +141,6 @@ CREATE TABLE item_versions(
version VARCHAR NOT NULL,
repo_iteration_id INTEGER NOT NULL,
definition TEXT NOT NULL,
- -- What privileges should be granted on pages where this
- -- resource/mapping is used.
- eval_allowed BOOLEAN NOT NULL,
- cors_bypass_allowed BOOLEAN NOT NULL,
UNIQUE (item_id, version, repo_iteration_id),
-- Allow foreign key from "mapping_statuses".
@@ -157,15 +153,19 @@ CREATE TABLE item_versions(
);
CREATE TABLE payloads(
- payload_id INTEGER PRIMARY KEY,
+ payload_id INTEGER PRIMARY KEY,
- mapping_item_id INTEGER NOT NULL,
- pattern VARCHAR NOT NULL,
+ mapping_item_id INTEGER NOT NULL,
+ pattern VARCHAR NOT NULL,
+ -- What privileges should be granted on pages where this
+ -- resource/mapping is used.
+ eval_allowed BOOLEAN NOT NULL,
+ cors_bypass_allowed BOOLEAN NOT NULL,
UNIQUE (mapping_item_id, pattern),
FOREIGN KEY (mapping_item_id)
- REFERENCES item_versions (versioned_item_id)
+ REFERENCES item_versions (item_version_id)
ON DELETE CASCADE
);
@@ -179,9 +179,11 @@ CREATE TABLE resolved_depended_resources(
PRIMARY KEY (payload_id, resource_item_id),
FOREIGN KEY (payload_id)
- REFERENCES payloads (payload_id),
+ REFERENCES payloads (payload_id)
+ ON DELETE CASCADE,
FOREIGN KEY (resource_item_id)
REFERENCES item_versions (item_version_id)
+ ON DELETE CASCADE
) WITHOUT ROWID;
-- CREATE TABLE resolved_required_mappings(