[builder][server][proxy] orchestrate Guix-powered development with a Makefile

* We're no longer treating builds from git and builds from release tarball differently. We now generate a source tarball under `./dist/` in both cases.
* We're now using `guix time-machine` to ensure we're always running a Guix revision that works.

5 files changed, 169 insertions, 168 deletions

diff --git a/.gitignore b/.gitignore
index e2b391c..63a911b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,6 @@ setuptools
 src/hydrilla/_version.py
 src/hydrilla/locales/messages.pot
 messages.mo
+make-release.log
+rel-tarball-repack
+haketilo-and-hydrilla-bin-*.tar.gz
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..03e4454
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: CC0-1.0
+
+# Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org>
+#
+# Available under the terms of Creative Commons Zero v1.0 Universal.
+
+GUIX := guix
+
+# Almost all commands in this Makefilo are run through `guix time-machine` with
+# Guix revision fixed to the one from the commit below. This ensures that the
+# same working environment is always used.
+GUIX_COMMIT := a86979b41a49a8fcdaa887970ba594dbba701226
+
+GUIX_TM = $(GUIX) time-machine --commit=$(GUIX_COMMIT) --
+
+GUIX_DEVSHELL = $(GUIX_TM) shell -Df guix.scm --
+
+wheel:
+	$(GUIX_DEVSHELL) python3 -m build
+
+dist src/hydrilla/_version.py:
+	$(GUIX_DEVSHELL) python3 -m build -s
+
+# Make a release tarball and repack its files as writeable - this will make it
+# easier for non-technical users to remove the unpacked release once they no
+# longer need it.
+release: clean-tarball-repack dist
+	$(GUIX_TM) pack -L ./hydrilla-guix -RR hydrilla \
+		-S /hydrilla=bin/hydrilla \
+		-S /hydrilla-builder=bin/hydrilla-builder \
+		-S /hydrilla-server=bin/hydrilla-server \
+		-S /haketilo=bin/haketilo | tee make-release.log
+	VER="$$(grep '^Version:' src/hydrilla.egg-info/PKG-INFO | cut -d' ' -f2)" && \
+		RELNAME=haketilo-and-hydrilla-bin-"$$VER"-"$$(arch)" && \
+		PACKFILE="$$(tail -1 make-release.log)" && \
+		mkdir rel-tarball-repack/ && \
+		mkdir rel-tarball-repack/"$$RELNAME" && \
+		tar -C rel-tarball-repack/"$$RELNAME"/ -xf "$$PACKFILE" && \
+		chmod -R +w rel-tarball-repack/"$$RELNAME" && \
+		tar -C rel-tarball-repack/ -cf "$$RELNAME".tar.gz "$$RELNAME"
+
+shell:
+	$(GUIX_DEVSHELL) || true
+
+shell-with-haketilo: dist
+	$(GUIX_TM) shell -f guix.scm -- || true
+
+catalogs:
+	$(GUIX_DEVSHELL) python3 setup.py compile_catalog
+
+refresh-catalogs:
+	$(GUIX_DEVSHELL) sh -c \
+		"python3 setup.py extract_messages && python3 setup.py update_catalog && python3 setup.py compile_catalog"
+
+test: src/hydrilla/_version.py catalogs
+	$(GUIX_DEVSHELL) pytest
+
+run-haketilo: src/hydrilla/_version.py catalogs
+	PYTHONPATH=./src $(GUIX_DEVSHELL) python3 -m hydrilla.mitmproxy_launcher
+
+clean-tarball-repack:
+	test -d rel-tarball-repack/ && chmod -R +w rel-tarball-repack/ || true
+	rm -rf rel-tarball-repack/
+
+clean: clean-tarball-repack
+	rm -rf build/ dist/ src/hydrilla.egg-info src/hydrilla/_version.py \
+		src/hydrilla/locales/messages.pot make-release.log \
+		haketilo-and-hydrilla-bin-*.tar.gz
+	find src/hydrilla/locales/ -name "messages.mo" -delete
+	rm -rf $$(find -name "__pycache__")
+
+.PHONY: release shell shell-with-haketilo dist wheel catalogs refresh-catalogs \
+	test run-haketilo clean-tarball-repack clean
diff --git a/README.md b/README.md
index fc1bda1..ae46b6a 100644
--- a/README.md
+++ b/README.md
@@ -7,143 +7,56 @@ Haketilo started as a browser extension (a WebExtension) with a dedicated user
 scripts repository server, Hydrilla. It has since been rewritten as an HTTP
 proxy. This repository contains the Python code of Hydrilla and Haketilo.
 
-## Getting started
+## Installation and running
 
-At the moment the recommended method of using Haketilo and Hydrilla is through
-the [GNU Guix](https://guix.gnu.org/) package manager. Installation from Python
-wheel is also possible but not described here.
+Head on to the relevant [wiki
+article](https://hydrillabugs.koszko.org/projects/haketilo/wiki/User_manual) for
+usage instructions for casual users.
 
-Build instructions have been most recently tested with Guix version
-1.3.0-26.fd00ac7.
+## Hacking
 
-### Building locally (from release tarball)
+At the moment the recommended method of hacking Haketilo and Hydrilla's codebase
+is through the [GNU Guix](https://guix.gnu.org/) package manager.
 
-Assuming GNU Guix is already installed and working, we can execute the following
-command in the project root to spawn a shell with Hydrilla & Haketilo available
-inside.
+A Guix version of at least 1.1.0 is expected. Most development tasks have been
+automated through Makefile. Rules use the `guix time-machine` command under the
+hood to pull and run a fixed Guix version that is known to work with the
+code. The most important make rules are described below.
 
-```
-guix environment -L . --ad-hoc -e '(@ (hydrilla) hydrilla)'
-```
+### `make wheel`
+Builds the project and put a source tarbal and a Python wheel under `dist/`.
 
-We can then make a test by running one of the programs, e.g.
-```
-haketilo --version
-```
+### `make dist`
+Generates a source tarbal under `dist/`.
 
-To instead install Haketilo & Hydrilla into the current Guix profile, treat your
-terminal with the following incantation
+### `make release`
+Produces a standalone, relocatable, binary release tarball that should work on
+most GNU/Linux systems. The tarball is written as
+`./haketilo-and-hydrilla-bin-<VERSION>-<ARCHITECTURE>.tar.gz`.
 
-```
-guix package -L . -e '(@ (hydrilla) hydrilla)'
-```
+### `make shell`
+Spawns a development shell with project's all dependencies and dev dependencies
+available.
 
-### Building locally (from git checkout)
+### `make shell-with-haketilo`
+Spawns a shell with Haketilo and Hydrilla built and available by means of a Guix
+profile.
 
-Due to some nuances of the setuptools-scm tool we're using, it is necessary to
-generate a project source tarball under `dist/` before building the Guix
-package.
+### `make catalogs`
+Rebuilds GNU gettext message catalogs used for localization.
 
-```
-guix environment -L . -e '(@ (hydrilla) hydrilla)' -- python3 -m build -s
-```
+### `make refresh-catalogs`
+Extracts messages form source files, refreshes the GNU gettext message catalog
+source files and rebuilds the catalogs.
 
-After that, we can start a shell with Hydrilla & Haketilo installed
+### `make test`
+Runs the automated test suite.
 
-```
-guix environment -L . --ad-hoc -e '(@ (hydrilla) hydrilla-dist-tarball)'
-```
+### `make run-haketilo`
+Runs `haketilo` program from sources.
 
-or just install to Guix profile
-
-```
-guix package -L . -e '(@ (hydrilla) hydrilla-dist-tarball)'
-```
-
-or build a binary package suitable for distribution to other GNU/Linux users
-
-```
-guix pack -L . -RR \
-    -S /hydrilla=bin/hydrilla \
-    -S /hydrilla-builder=bin/hydrilla-builder \
-    -S /hydrilla-server=bin/hydrilla-server \
-    -S /haketilo=bin/haketilo \
-    -e '(@ (hydrilla) hydrilla-dist-tarball)'
-```
-
-### Running from source
-
-During development, it is convenient to run the tools being worked on without
-putting them in a package. To spawn a shell with all development dependencies
-installed, run
-
-```
-guix environment -L . -e '(@ (hydrilla) hydrilla)'
-```
-
-For software to run, we first need to compile message catalogs and make sure the
-relevant metadata has been extracted from git by setuptools-scm. Inside the
-shell we just spawned, we run
-
-```
-# The following command, besides building a source tarball, generates the
-# src/hydrilla/_version.py file we need.
-python3 -m build -s
-# Generate .mo file(s) for gettext.
-./setup.py compile_catalog
-```
-
-Tools can be manually tested by telling Python interpreter tu execute the
-relevant module, e.g
-
-```
-PYTHONPATH=./src/ python3 -m hydrilla.mitmproxy_launcher --version
-```
-
-#### Running tests
-
-Hydrilla uses pytest. Tests can be run with
-
-```
-pytest
-```
-
-Please refer to the
-[pytest documentation](https://docs.pytest.org/en/stable/how-to/usage.html) for
-more details.
-
-#### Working on message catalogs
-
-There are 3 commands we'll want to use.
-
-```
-# Generate a message catalog template (src/hydrilla/locales/messages.pot)
-./setup.py extract_messages
-# Merge the generated template into existing .po catalog file(s)
-./setup.py update_catalog
-# Generate .mo file(s) that will be loaded by gettext
-./setup.py compile_catalog
-```
-
-Please refer to the
-[Babel documentation](https://babel.pocoo.org/en/latest/messages.html#message-extraction)
-for more details.
-
-#### Exiting Guix environment
-
-Once we're done hacking on the project, we can type
-
-```
-exit
-```
-
-in the shell... or just hit `Ctrl+d`.
-
-### User documentation
-
-Please look at
-[our wiki](https://hydrillabugs.koszko.org/projects/haketilo/wiki) for
-instructions on how to operate Haketilo and Hydrilla.
+### `make clean`
+Removes generated files.
 
 ## Contributing, asking for help, giving feedback, reporting bugs
 
diff --git a/guix.scm b/guix.scm
new file mode 100644
index 0000000..10a7cec
--- /dev/null
+++ b/guix.scm
@@ -0,0 +1,16 @@
+;; SPDX-License-Identifier: CC0-1.0
+
+;; Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org>
+;;
+;; Available under the terms of Creative Commons Zero v1.0 Universal.
+
+;; We need the actual hydrilla package to be defined in a module so that we can
+;; specify it to guix-pack. Hence, the bulk of definitions resides in
+;; `hydrilla-guix/hydrilla.scm` and this file is just a thin wrapper around
+;; that.
+
+(add-to-load-path (string-append (dirname (current-filename)) "/hydrilla-guix"))
+
+(use-modules (hydrilla))
+
+hydrilla
diff --git a/hydrilla.scm b/hydrilla-guix/hydrilla.scm
index 6b8ee5c..f580942 100644
--- a/hydrilla.scm
+++ b/hydrilla-guix/hydrilla.scm
@@ -10,10 +10,11 @@
  (ice-9 rdelim)
  (ice-9 regex))
 
-(define %source-dir (dirname (current-filename)))
+;; We're mostly running from Makefile, so it's pretty safe to use `getcwd`.
+(define %source-dir (getcwd))
 
 ;; The PKG-INFO file is generated when running `python3 -m build -s` or similar.
-;; It is also automatically included in the generated tarballs.
+;; It is also automatically included in the source release tarballs.
 (define %hydrilla-version
   (if (access? "src/hydrilla.egg-info/PKG-INFO" R_OK)
       (let* ((port (open-input-file "src/hydrilla.egg-info/PKG-INFO"))
@@ -26,6 +27,9 @@
         (process-line process-line))
       "unknown"))
 
+(define source-tarball-name
+  (string-append "hydrilla-" %hydrilla-version ".tar.gz"))
+
 (use-modules
  (guix packages)
  (guix download)
@@ -109,7 +113,7 @@ network stream packet formats, etc.")
       (version "22.0.0")
       (source
        (origin
-         (inherit (package-source base))
+         (method url-fetch)
          (uri (pypi-uri "pyOpenSSL" version))
          (sha256
           (base32
@@ -233,35 +237,35 @@ replay web traffic such as HTTP/1, HTTP/2, WebSockets, or any other
 SSL/TLS-protected protocols.")
     (license license:expat)))
 
-(define-public python-immutables-for-haketilo
-  (let ((base python-immutables))
-    (package
-      (inherit base)
-      (version "0.19")
-      (source
-       (origin
-         ;; Old version tarballs seem to be getting deleted from PyPI each time
-         ;; a new version comes out.
-         (method git-fetch)
-         (uri (git-reference
-               (url "https://github.com/MagicStack/immutables")
-               (commit (string-append "v" version))))
-         (sha256
-          (base32
-           "1awjylp4pl0jknwgnabg7kkj8f8883fkf99spsdrw1pj1acajvy9"))))
-      (arguments
-       `(#:phases
-         (modify-phases %standard-phases
-           (add-after 'unpack 'fix-expected-mypy-types
-             (lambda _
-               (substitute* "tests/test-data/check-immu.test"
-                 (("builtins.str") "builtins.str*"))))
-           (replace 'check
-             (lambda* (#:key tests? #:allow-other-keys)
-               (when tests?
-                 (invoke "pytest")))))))
-      (native-inputs
-       (list python-pytest python-mypy)))))
+;; (define-public python-immutables-for-haketilo
+;;   (let ((base python-immutables))
+;;     (package
+;;       (inherit base)
+;;       (version "0.19")
+;;       (source
+;;        (origin
+;;          ;; Old version tarballs seem to be getting deleted from PyPI each time
+;;          ;; a new version comes out.
+;;          (method git-fetch)
+;;          (uri (git-reference
+;;                (url "https://github.com/MagicStack/immutables")
+;;                (commit (string-append "v" version))))
+;;          (sha256
+;;           (base32
+;;            "1awjylp4pl0jknwgnabg7kkj8f8883fkf99spsdrw1pj1acajvy9"))))
+;;       (arguments
+;;        `(#:phases
+;;          (modify-phases %standard-phases
+;;            (add-after 'unpack 'fix-expected-mypy-types
+;;              (lambda _
+;;                (substitute* "tests/test-data/check-immu.test"
+;;                  (("builtins.str") "builtins.str*"))))
+;;            (replace 'check
+;;              (lambda* (#:key tests? #:allow-other-keys)
+;;                (when tests?
+;;                  (invoke "pytest")))))))
+;;       (native-inputs
+;;        (list python-pytest python-mypy)))))
 
 (define-public python-types-requests
   (package
@@ -286,7 +290,10 @@ static types.")
   (package
     (name "hydrilla")
     (version %hydrilla-version)
-    (source (local-file %source-dir #:recursive? #t))
+    (source
+     ;; setuptools_scm makes it impossible to build directly from git
+     ;; checkout. We instead build from source tarball generated under ./dist/.
+     (local-file (string-append %source-dir "/dist/" source-tarball-name)))
     (build-system python-build-system)
     (arguments
      `(#:phases
@@ -302,7 +309,7 @@ static types.")
            python-flask-for-haketilo
            python-gnupg
            python-html5lib
-           python-immutables-for-haketilo
+           python-immutables ;;python-immutables-for-haketilo
            python-itsdangerous
            python-jsonschema
            reuse))
@@ -320,14 +327,3 @@ having their original JavaScript replaced by user-provided scripts. Haketilo
 combines the functionalities of content blocker and user script manager. It can
 be used with its script repository, Hydrilla.")
     (license (list license:agpl3+ license:gpl3+ license:cc0))))
-
-;; Use this variant when building from a tarball generated under dist/. This
-;; can be used to build from a git checkout after running `python3 -m build -s`
-;; or similar.
-(define-public hydrilla-dist-tarball
-  (let ((base hydrilla)
-        (filename (string-append "hydrilla-" %hydrilla-version ".tar.gz")))
-    (package
-      (inherit base)
-      (source (local-file
-               (string-append %source-dir "/dist/" filename))))))