diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | Makefile | 73 | ||||
| -rw-r--r-- | README.md | 159 | ||||
| -rw-r--r-- | guix.scm | 16 | ||||
| -rw-r--r-- | hydrilla-guix/hydrilla.scm (renamed from hydrilla.scm) | 86 |
5 files changed, 169 insertions, 168 deletions
@@ -12,3 +12,6 @@ setuptools src/hydrilla/_version.py src/hydrilla/locales/messages.pot messages.mo +make-release.log +rel-tarball-repack +haketilo-and-hydrilla-bin-*.tar.gz diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..03e4454 --- /dev/null +++ b/Makefile @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: CC0-1.0 + +# Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> +# +# Available under the terms of Creative Commons Zero v1.0 Universal. + +GUIX := guix + +# Almost all commands in this Makefilo are run through `guix time-machine` with +# Guix revision fixed to the one from the commit below. This ensures that the +# same working environment is always used. +GUIX_COMMIT := a86979b41a49a8fcdaa887970ba594dbba701226 + +GUIX_TM = $(GUIX) time-machine --commit=$(GUIX_COMMIT) -- + +GUIX_DEVSHELL = $(GUIX_TM) shell -Df guix.scm -- + +wheel: + $(GUIX_DEVSHELL) python3 -m build + +dist src/hydrilla/_version.py: + $(GUIX_DEVSHELL) python3 -m build -s + +# Make a release tarball and repack its files as writeable - this will make it +# easier for non-technical users to remove the unpacked release once they no +# longer need it. +release: clean-tarball-repack dist + $(GUIX_TM) pack -L ./hydrilla-guix -RR hydrilla \ + -S /hydrilla=bin/hydrilla \ + -S /hydrilla-builder=bin/hydrilla-builder \ + -S /hydrilla-server=bin/hydrilla-server \ + -S /haketilo=bin/haketilo | tee make-release.log + VER="$$(grep '^Version:' src/hydrilla.egg-info/PKG-INFO | cut -d' ' -f2)" && \ + RELNAME=haketilo-and-hydrilla-bin-"$$VER"-"$$(arch)" && \ + PACKFILE="$$(tail -1 make-release.log)" && \ + mkdir rel-tarball-repack/ && \ + mkdir rel-tarball-repack/"$$RELNAME" && \ + tar -C rel-tarball-repack/"$$RELNAME"/ -xf "$$PACKFILE" && \ + chmod -R +w rel-tarball-repack/"$$RELNAME" && \ + tar -C rel-tarball-repack/ -cf "$$RELNAME".tar.gz "$$RELNAME" + +shell: + $(GUIX_DEVSHELL) || true + +shell-with-haketilo: dist + $(GUIX_TM) shell -f guix.scm -- || true + +catalogs: + $(GUIX_DEVSHELL) python3 setup.py compile_catalog + +refresh-catalogs: + $(GUIX_DEVSHELL) sh -c \ + "python3 setup.py extract_messages && python3 setup.py update_catalog && python3 setup.py compile_catalog" + +test: src/hydrilla/_version.py catalogs + $(GUIX_DEVSHELL) pytest + +run-haketilo: src/hydrilla/_version.py catalogs + PYTHONPATH=./src $(GUIX_DEVSHELL) python3 -m hydrilla.mitmproxy_launcher + +clean-tarball-repack: + test -d rel-tarball-repack/ && chmod -R +w rel-tarball-repack/ || true + rm -rf rel-tarball-repack/ + +clean: clean-tarball-repack + rm -rf build/ dist/ src/hydrilla.egg-info src/hydrilla/_version.py \ + src/hydrilla/locales/messages.pot make-release.log \ + haketilo-and-hydrilla-bin-*.tar.gz + find src/hydrilla/locales/ -name "messages.mo" -delete + rm -rf $$(find -name "__pycache__") + +.PHONY: release shell shell-with-haketilo dist wheel catalogs refresh-catalogs \ + test run-haketilo clean-tarball-repack clean @@ -7,143 +7,56 @@ Haketilo started as a browser extension (a WebExtension) with a dedicated user scripts repository server, Hydrilla. It has since been rewritten as an HTTP proxy. This repository contains the Python code of Hydrilla and Haketilo. -## Getting started +## Installation and running -At the moment the recommended method of using Haketilo and Hydrilla is through -the [GNU Guix](https://guix.gnu.org/) package manager. Installation from Python -wheel is also possible but not described here. +Head on to the relevant [wiki +article](https://hydrillabugs.koszko.org/projects/haketilo/wiki/User_manual) for +usage instructions for casual users. -Build instructions have been most recently tested with Guix version -1.3.0-26.fd00ac7. +## Hacking -### Building locally (from release tarball) +At the moment the recommended method of hacking Haketilo and Hydrilla's codebase +is through the [GNU Guix](https://guix.gnu.org/) package manager. -Assuming GNU Guix is already installed and working, we can execute the following -command in the project root to spawn a shell with Hydrilla & Haketilo available -inside. +A Guix version of at least 1.1.0 is expected. Most development tasks have been +automated through Makefile. Rules use the `guix time-machine` command under the +hood to pull and run a fixed Guix version that is known to work with the +code. The most important make rules are described below. -``` -guix environment -L . --ad-hoc -e '(@ (hydrilla) hydrilla)' -``` +### `make wheel` +Builds the project and put a source tarbal and a Python wheel under `dist/`. -We can then make a test by running one of the programs, e.g. -``` -haketilo --version -``` +### `make dist` +Generates a source tarbal under `dist/`. -To instead install Haketilo & Hydrilla into the current Guix profile, treat your -terminal with the following incantation +### `make release` +Produces a standalone, relocatable, binary release tarball that should work on +most GNU/Linux systems. The tarball is written as +`./haketilo-and-hydrilla-bin-<VERSION>-<ARCHITECTURE>.tar.gz`. -``` -guix package -L . -e '(@ (hydrilla) hydrilla)' -``` +### `make shell` +Spawns a development shell with project's all dependencies and dev dependencies +available. -### Building locally (from git checkout) +### `make shell-with-haketilo` +Spawns a shell with Haketilo and Hydrilla built and available by means of a Guix +profile. -Due to some nuances of the setuptools-scm tool we're using, it is necessary to -generate a project source tarball under `dist/` before building the Guix -package. +### `make catalogs` +Rebuilds GNU gettext message catalogs used for localization. -``` -guix environment -L . -e '(@ (hydrilla) hydrilla)' -- python3 -m build -s -``` +### `make refresh-catalogs` +Extracts messages form source files, refreshes the GNU gettext message catalog +source files and rebuilds the catalogs. -After that, we can start a shell with Hydrilla & Haketilo installed +### `make test` +Runs the automated test suite. -``` -guix environment -L . --ad-hoc -e '(@ (hydrilla) hydrilla-dist-tarball)' -``` +### `make run-haketilo` +Runs `haketilo` program from sources. -or just install to Guix profile - -``` -guix package -L . -e '(@ (hydrilla) hydrilla-dist-tarball)' -``` - -or build a binary package suitable for distribution to other GNU/Linux users - -``` -guix pack -L . -RR \ - -S /hydrilla=bin/hydrilla \ - -S /hydrilla-builder=bin/hydrilla-builder \ - -S /hydrilla-server=bin/hydrilla-server \ - -S /haketilo=bin/haketilo \ - -e '(@ (hydrilla) hydrilla-dist-tarball)' -``` - -### Running from source - -During development, it is convenient to run the tools being worked on without -putting them in a package. To spawn a shell with all development dependencies -installed, run - -``` -guix environment -L . -e '(@ (hydrilla) hydrilla)' -``` - -For software to run, we first need to compile message catalogs and make sure the -relevant metadata has been extracted from git by setuptools-scm. Inside the -shell we just spawned, we run - -``` -# The following command, besides building a source tarball, generates the -# src/hydrilla/_version.py file we need. -python3 -m build -s -# Generate .mo file(s) for gettext. -./setup.py compile_catalog -``` - -Tools can be manually tested by telling Python interpreter tu execute the -relevant module, e.g - -``` -PYTHONPATH=./src/ python3 -m hydrilla.mitmproxy_launcher --version -``` - -#### Running tests - -Hydrilla uses pytest. Tests can be run with - -``` -pytest -``` - -Please refer to the -[pytest documentation](https://docs.pytest.org/en/stable/how-to/usage.html) for -more details. - -#### Working on message catalogs - -There are 3 commands we'll want to use. - -``` -# Generate a message catalog template (src/hydrilla/locales/messages.pot) -./setup.py extract_messages -# Merge the generated template into existing .po catalog file(s) -./setup.py update_catalog -# Generate .mo file(s) that will be loaded by gettext -./setup.py compile_catalog -``` - -Please refer to the -[Babel documentation](https://babel.pocoo.org/en/latest/messages.html#message-extraction) -for more details. - -#### Exiting Guix environment - -Once we're done hacking on the project, we can type - -``` -exit -``` - -in the shell... or just hit `Ctrl+d`. - -### User documentation - -Please look at -[our wiki](https://hydrillabugs.koszko.org/projects/haketilo/wiki) for -instructions on how to operate Haketilo and Hydrilla. +### `make clean` +Removes generated files. ## Contributing, asking for help, giving feedback, reporting bugs diff --git a/guix.scm b/guix.scm new file mode 100644 index 0000000..10a7cec --- /dev/null +++ b/guix.scm @@ -0,0 +1,16 @@ +;; SPDX-License-Identifier: CC0-1.0 + +;; Copyright (C) 2022 Wojtek Kosior <koszko@koszko.org> +;; +;; Available under the terms of Creative Commons Zero v1.0 Universal. + +;; We need the actual hydrilla package to be defined in a module so that we can +;; specify it to guix-pack. Hence, the bulk of definitions resides in +;; `hydrilla-guix/hydrilla.scm` and this file is just a thin wrapper around +;; that. + +(add-to-load-path (string-append (dirname (current-filename)) "/hydrilla-guix")) + +(use-modules (hydrilla)) + +hydrilla diff --git a/hydrilla.scm b/hydrilla-guix/hydrilla.scm index 6b8ee5c..f580942 100644 --- a/hydrilla.scm +++ b/hydrilla-guix/hydrilla.scm @@ -10,10 +10,11 @@ (ice-9 rdelim) (ice-9 regex)) -(define %source-dir (dirname (current-filename))) +;; We're mostly running from Makefile, so it's pretty safe to use `getcwd`. +(define %source-dir (getcwd)) ;; The PKG-INFO file is generated when running `python3 -m build -s` or similar. -;; It is also automatically included in the generated tarballs. +;; It is also automatically included in the source release tarballs. (define %hydrilla-version (if (access? "src/hydrilla.egg-info/PKG-INFO" R_OK) (let* ((port (open-input-file "src/hydrilla.egg-info/PKG-INFO")) @@ -26,6 +27,9 @@ (process-line process-line)) "unknown")) +(define source-tarball-name + (string-append "hydrilla-" %hydrilla-version ".tar.gz")) + (use-modules (guix packages) (guix download) @@ -109,7 +113,7 @@ network stream packet formats, etc.") (version "22.0.0") (source (origin - (inherit (package-source base)) + (method url-fetch) (uri (pypi-uri "pyOpenSSL" version)) (sha256 (base32 @@ -233,35 +237,35 @@ replay web traffic such as HTTP/1, HTTP/2, WebSockets, or any other SSL/TLS-protected protocols.") (license license:expat))) -(define-public python-immutables-for-haketilo - (let ((base python-immutables)) - (package - (inherit base) - (version "0.19") - (source - (origin - ;; Old version tarballs seem to be getting deleted from PyPI each time - ;; a new version comes out. - (method git-fetch) - (uri (git-reference - (url "https://github.com/MagicStack/immutables") - (commit (string-append "v" version)))) - (sha256 - (base32 - "1awjylp4pl0jknwgnabg7kkj8f8883fkf99spsdrw1pj1acajvy9")))) - (arguments - `(#:phases - (modify-phases %standard-phases - (add-after 'unpack 'fix-expected-mypy-types - (lambda _ - (substitute* "tests/test-data/check-immu.test" - (("builtins.str") "builtins.str*")))) - (replace 'check - (lambda* (#:key tests? #:allow-other-keys) - (when tests? - (invoke "pytest"))))))) - (native-inputs - (list python-pytest python-mypy))))) +;; (define-public python-immutables-for-haketilo +;; (let ((base python-immutables)) +;; (package +;; (inherit base) +;; (version "0.19") +;; (source +;; (origin +;; ;; Old version tarballs seem to be getting deleted from PyPI each time +;; ;; a new version comes out. +;; (method git-fetch) +;; (uri (git-reference +;; (url "https://github.com/MagicStack/immutables") +;; (commit (string-append "v" version)))) +;; (sha256 +;; (base32 +;; "1awjylp4pl0jknwgnabg7kkj8f8883fkf99spsdrw1pj1acajvy9")))) +;; (arguments +;; `(#:phases +;; (modify-phases %standard-phases +;; (add-after 'unpack 'fix-expected-mypy-types +;; (lambda _ +;; (substitute* "tests/test-data/check-immu.test" +;; (("builtins.str") "builtins.str*")))) +;; (replace 'check +;; (lambda* (#:key tests? #:allow-other-keys) +;; (when tests? +;; (invoke "pytest"))))))) +;; (native-inputs +;; (list python-pytest python-mypy))))) (define-public python-types-requests (package @@ -286,7 +290,10 @@ static types.") (package (name "hydrilla") (version %hydrilla-version) - (source (local-file %source-dir #:recursive? #t)) + (source + ;; setuptools_scm makes it impossible to build directly from git + ;; checkout. We instead build from source tarball generated under ./dist/. + (local-file (string-append %source-dir "/dist/" source-tarball-name))) (build-system python-build-system) (arguments `(#:phases @@ -302,7 +309,7 @@ static types.") python-flask-for-haketilo python-gnupg python-html5lib - python-immutables-for-haketilo + python-immutables ;;python-immutables-for-haketilo python-itsdangerous python-jsonschema reuse)) @@ -320,14 +327,3 @@ having their original JavaScript replaced by user-provided scripts. Haketilo combines the functionalities of content blocker and user script manager. It can be used with its script repository, Hydrilla.") (license (list license:agpl3+ license:gpl3+ license:cc0)))) - -;; Use this variant when building from a tarball generated under dist/. This -;; can be used to build from a git checkout after running `python3 -m build -s` -;; or similar. -(define-public hydrilla-dist-tarball - (let ((base hydrilla) - (filename (string-append "hydrilla-" %hydrilla-version ".tar.gz"))) - (package - (inherit base) - (source (local-file - (string-append %source-dir "/dist/" filename)))))) |