fill served definitions with computed sha256 sums of files

3 files changed, 112 insertions, 50 deletions

diff --git a/src/pydrilla/pydrilla.py b/src/pydrilla/pydrilla.py
index b0a5974..9e697ba 100644
--- a/src/pydrilla/pydrilla.py
+++ b/src/pydrilla/pydrilla.py
@@ -28,7 +28,7 @@ from flask import Flask, Blueprint, current_app, url_for, abort, request, \
     redirect
 from jinja2 import Environment, PackageLoader
 import re
-#from hashlib import sha256
+from hashlib import sha256
 import os
 import pathlib
 import json
@@ -333,17 +333,52 @@ class MappingItem:
 
 class Content:
     '''Stores serveable website content.'''
-    def __init__(self):
+    def __init__(self, content_dir_path):
+        '''
+        When an instance of Content is constructed, it searches
+        content_dir_path for custom serveable site content and loads it.
+        '''
         self.resources = {}
         self.mappings  = {}
         self.licenses  = {}
         self.indexes   = {}
         self.definition_processors = {
-            'resource': self.process_resource_or_mapping,
-            'mapping': self.process_resource_or_mapping,
-            'license': self.process_license
+            'resource': self._process_resource_or_mapping,
+            'mapping': self._process_resource_or_mapping,
+            'license': self._process_license
         }
         self.patterns_by_proto = {}
+        self.file_sha256sums = {}
+
+        self.content_dir_path = pathlib.Path(content_dir_path).resolve()
+
+        if not self.content_dir_path.is_dir():
+            raise ValueError(_('content_dir_path_not_dir'))
+
+        for subdir_path in self.content_dir_path.iterdir():
+            if not subdir_path.is_dir():
+                continue
+            try:
+                self._load_content_from_subdir(subdir_path, subdir_path.name)
+            except Exception as e:
+                if current_app._pydrilla_werror:
+                    raise e from None
+                logging.error(_('couldnt_load_content_from_%s'), subdir_path,
+                              exc_info=True)
+
+        self._report_missing()
+        self._finalize()
+
+    def _load_content_from_subdir(self, subdir_path, source_name):
+        '''
+        Helper function used to load definitions from index.json of a
+        subdirectory of the content direcotory.
+        '''
+        index_path = subdir_path / 'index.json'
+        with open(index_path) as index_file:
+            index = json.loads(strip_json_comments(index_file.read()))
+
+        self._process_index(index, source_name)
 
     @staticmethod
     def register_item(dict, item):
@@ -361,13 +396,38 @@ class Content:
 
     @staticmethod
     def _process_copyright_and_license(definition):
-        '''Helper function used by other process_*() methods.'''
+        '''Helper function used by other _process_*() methods.'''
         for field in ['copyright', 'licenses']:
             if definition[field] == 'auto':
                 raise MyNotImplError(f'"{{field}}": "auto"',
                                      definition['source_name'])
 
-    def process_resource_or_mapping(self, definition, index):
+    def _get_file_sha256sum(self, path):
+        '''
+        Compute sha256 of the file at path. Cache results on this Content
+        object.
+        '''
+        path = path.resolve()
+        sha256sum = self.file_sha256sums.get(path)
+
+        if sha256sum is None:
+            with open(path, mode='rb') as hashed_file:
+                sha256sum = sha256(hashed_file.read()).digest().hex()
+            self.file_sha256sums[path] = sha256sum
+
+        return sha256sum
+
+    def _add_file_sha256sum(self, source_name, file_object):
+        '''
+        Expect file_object to be a dict with field "file" holding a file path
+        relative to content directory's subdirectory source_name. Compute or
+        fetch from cache the sha256 sum of that file and put it in file_object's
+        "sha256" field.
+        '''
+        file_path = self.content_dir_path / source_name / file_object['file']
+        file_object['sha256'] = self._get_file_sha256sum(file_path)
+
+    def _process_resource_or_mapping(self, definition, index):
         '''
         Sanitizes, autocompletes and registers serveable mapping/resource
         definition.
@@ -378,10 +438,13 @@ class Content:
             self._process_copyright_and_license(definition)
             definition['dependencies'] = definition.get('dependencies', [])
             self.register_item(self.resources, definition)
+            source_name = definition['source_name']
+            for script in definition['scripts']:
+                self._add_file_sha256sum(source_name, script)
         else:
             self.register_item(self.mappings, definition)
 
-    def process_license(self, license, index):
+    def _process_license(self, license, index):
         '''Sanitizes and registers serveable license definition.'''
         identifier = license['identifier']
         if identifier in self.licenses:
@@ -389,7 +452,15 @@ class Content:
 
         self.licenses[identifier] = license
 
-    def process_index(self, index, source_name):
+        source_name = license['source_name']
+        for legal_text in license['legal_text']:
+            self._add_file_sha256sum(source_name, legal_text)
+
+        notice = license.get('notice')
+        if notice is not None:
+            self._add_file_sha256sum(source_name, notice)
+
+    def _process_index(self, index, source_name):
         '''
         Sanitizes, autocompletes and registers data from a loaded index.json
         file.
@@ -429,7 +500,7 @@ class Content:
             for item in versioned_item.by_version.values():
                 yield item
 
-    def report_missing(self):
+    def _report_missing(self):
         '''
         Use logger to print information about items that are referenced but
         were not loaded.
@@ -488,7 +559,7 @@ class Content:
                 if payload not in self.resources:
                     report_missing_payload(mapping, payload)
 
-    def finalize(self):
+    def _finalize(self):
         '''
         Initialize structures needed to serve queries. Called once after all
         data gets loaded.
@@ -543,35 +614,6 @@ class Content:
 
         return list(mappings.values())
 
-def load_content_from_subdir(subdir_path, source_name, content):
-    index_path = subdir_path / 'index.json'
-    with open(index_path) as index_file:
-        index = json.loads(strip_json_comments(index_file.read()))
-
-    content.process_index(index, source_name)
-
-def load_content(path):
-    if not path.is_dir():
-        raise ValueError(_('content_dir_path_not_dir'))
-
-    content = Content()
-
-    for subdir_path in path.iterdir():
-        if not subdir_path.is_dir():
-            continue
-        try:
-            load_content_from_subdir(subdir_path, subdir_path.name, content)
-        except Exception as e:
-            if current_app._pydrilla_werror:
-                raise e from None
-            logging.error(_('couldnt_load_content_from_%s'), subdir_path,
-                          exc_info=True)
-
-    content.report_missing()
-    content.finalize()
-
-    return content
-
 def create_app(config_path=(here / 'config.json'), flask_config={}):
     app = Flask(__package__)
     app.config.update(flask_config)
@@ -603,7 +645,7 @@ def create_app(config_path=(here / 'config.json'), flask_config={}):
     if not content_dir.is_absolute():
         content_dir = config_path.parent / content_dir
     with app.app_context():
-        app._pydrilla_content = load_content(content_dir.resolve())
+        app._pydrilla_content = Content(content_dir.resolve())
 
     app.register_blueprint(bp)
 
diff --git a/src/test/example_content/hello/index.json b/src/test/example_content/hello/index.json
index 12105c2..16843cb 100644
--- a/src/test/example_content/hello/index.json
+++ b/src/test/example_content/hello/index.json
@@ -150,8 +150,9 @@
 	    // Array of javascript files that belong to this resource.
 	    "scripts": [
 		{
-		    // Script name. It should also be a valid file path.
-		    "name": "hello.js",
+		    // Script name. It should also be a valid file path relative
+		    // to index.json's containing directory.
+		    "file": "hello.js",
 		    // Copyright and license info of a script file can be
 		    // specified using the same format as in the case of the
 		    // index.json file itself. If "copyright" or "license" is
@@ -160,7 +161,7 @@
 		    "copyright": "auto",
 		    "licenses":  "auto"
 		}, {
-		    "name":   "bye.js"
+		    "file":   "bye.js"
 		}
 	    ]
 	}, {
@@ -175,7 +176,7 @@
 	    "licenses":    "CC0-1.0",
 	    // If "dependencies" is empty, it can also be omitted.
 	    // "dependencies": [],
-	    "scripts": [{"name": "message.js"}]
+	    "scripts": [{"file": "message.js"}]
 	}, {
 	    "type": "mapping",
 
@@ -262,14 +263,16 @@
 	    //
 	    // "comment": "Expat license is the most common form of the license often called \"MIT\". Many other forms of \"MIT\" license exist. Here the name \"Expat\" is used to avoid ambiguity."
 
-	    // If applicable, a "notice" can be included. It shall then be a
-	    // path (relative to index.json) to a plain text file with that
-	    // notice.
+	    // If applicable, a "notice" can be included. It shall then be an
+	    // object with "file" field containing a path (relative to
+	    // index.json's directory) to a plain text file with that notice.
 	    //
-	    // "notice": "license-notice.txt"
+	    // "notice": {
+	    //     "file": "license-notice.txt"
+	    // }
 	    //
 	    // This is needed for example in case of GNU licenses (both with and
-	    // without exceptions). For example,
+	    // without exceptions). For instance,
 	    // "GPL-3.0-or-later-with-html-exception" could have the following
 	    // in its notice file:
 	    //
diff --git a/src/test/test_pydrilla.py b/src/test/test_pydrilla.py
index 22022ae..50757a7 100644
--- a/src/test/test_pydrilla.py
+++ b/src/test/test_pydrilla.py
@@ -28,6 +28,7 @@ import pytest
 import sys
 import shutil
 from pathlib import Path
+from hashlib import sha256
 from os import mkdir, unlink, environ
 import json
 from markupsafe import escape
@@ -37,6 +38,7 @@ from pydrilla import pydrilla, create_app
 test_dir         = Path(__file__).resolve().parent
 packages_dir     = test_dir.parent
 development_config_path = test_dir / 'development_config.json'
+example_content_dir = test_dir / 'example_content'
 
 @pytest.fixture
 def client():
@@ -51,6 +53,11 @@ def development_config():
         yield json.loads(pydrilla.strip_json_comments(config_file.read()))
 
 def test_api_basic(client, development_config):
+    def verify_sha256sum(source_name, file_object):
+        with open(example_content_dir / source_name / file_object['file'],
+                  mode='rb') as file:
+            assert sha256(file.read()).digest().hex() == file_object['sha256']
+
     response = client.get('/')
     assert b'html' in response.data
     sources_uri = development_config['hydrilla_sources_uri']
@@ -63,6 +70,11 @@ def test_api_basic(client, development_config):
         assert definition['type']        == item_type
         assert definition['source_name'] == 'hello'
         assert definition['version']     == [2021, 11, 10]
+        if item_type == 'resource':
+            assert type(definition['scripts']) is list
+            assert len(definition['scripts']) > 0
+            for script_file in definition['scripts']:
+                verify_sha256sum(definition['source_name'], script_file)
 
         response = client.get(f'/{item_type}s/helloapple?ver=2021.11.10.0')
         assert response.status_code == 200
@@ -91,6 +103,11 @@ def test_api_basic(client, development_config):
     assert definition['long_name']   == 'Creative Commons Zero v1.0 Universal'
     assert definition['source_name'] == 'hello'
 
+    assert type(definition['legal_text']) is list
+    assert len(definition['legal_text']) > 0
+    for license_file in definition['legal_text']:
+        verify_sha256sum(definition['source_name'], license_file)
+
     response = client.get('/licenses/random-bad-identifier')
     assert response.status_code == 404