aboutsummaryrefslogtreecommitdiff
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2013-2020, 2022 Ludovic Courtès <ludo@gnu.org>
;;; Copyright © 2016 Chris Marusich <cmmarusich@gmail.com>
;;; Copyright © 2022 Maxim Cournoyer <maxim.cournoyer@gmail.com>
;;; Copyright © 2024 Nicolas Graves <ngraves@ngraves.fr>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.

(define-module (gnu build install)
  #:use-module (guix build syscalls)
  #:use-module (guix build utils)
  #:use-module (guix build store-copy)
  #:use-module (srfi srfi-26)
  #:use-module (ice-9 match)
  #:export (install-boot-config
            evaluate-populate-directive
            populate-root-file-system
            install-database-and-gc-roots
            populate-single-profile-directory
            mount-cow-store
            unmount-cow-store))

;;; Commentary:
;;;
;;; This module supports the installation of the GNU system on a hard disk.
;;; It is meant to be used both in a build environment (in derivations that
;;; build VM images), and on the bare metal (when really installing the
;;; system.)
;;;
;;; Code:

(define (install-boot-config bootcfg bootcfg-location mount-point)
  "Atomically copy BOOTCFG into BOOTCFG-LOCATION on the MOUNT-POINT.  Note
that the caller must make sure that BOOTCFG is registered as a GC root so
that the fonts, background images, etc. referred to by BOOTCFG are not GC'd."
  (let* ((target (string-append mount-point bootcfg-location))
         (pivot  (string-append target ".new")))
    (mkdir-p (dirname target))

    ;; Copy BOOTCFG instead of just symlinking it, because symlinks won't
    ;; work when /boot is on a separate partition.  Do that atomically.
    (copy-file bootcfg pivot)
    (rename-file pivot target)))

(define* (evaluate-populate-directive directive target
                                      #:key
                                      (default-gid 0)
                                      (default-uid 0)
                                      (error-on-dangling-symlink? #t))
  "Evaluate DIRECTIVE, an sexp describing a file or directory to create under
directory TARGET.  DEFAULT-UID and DEFAULT-GID are the default UID and GID in
the context of the caller.  If the directive matches those defaults then,
'chown' won't be run.  When ERROR-ON-DANGLING-SYMLINK? is true, abort with an
error when a dangling symlink would be created."
  (define target* (if (string-suffix? "/" target)
                      target
                      (string-append target "/")))
  (let loop ((directive directive))
    (catch 'system-error
      (lambda ()
        (match directive
          (('directory name)
           (mkdir-p (string-append target* name)))
          (('directory name uid gid)
           (let ((dir (string-append target* name)))
             (mkdir-p dir)
             ;; If called from a context without "root" permissions, "chown"
             ;; to root will fail.  In that case, do not try to run "chown"
             ;; and assume that the file will be chowned elsewhere (when
             ;; interned in the store for instance).
             (or (and (= uid default-uid) (= gid default-gid))
                 (chown dir uid gid))))
          (('directory name uid gid mode)
           (loop `(directory ,name ,uid ,gid))
           (chmod (string-append target* name) mode))
          (('file name)
           (call-with-output-file (string-append target* name)
             (const #t)))
          (('file name (? string? content))
           (call-with-output-file (string-append target* name)
             (lambda (port)
               (display content port))))
          ((new '-> old)
           (let ((new* (string-append target* new)))
             (let try ()
               (catch 'system-error
                 (lambda ()
                   (when error-on-dangling-symlink?
                     ;; When the symbolic link points to a relative path,
                     ;; checking if its target exists must be done relatively
                     ;; to the link location.
                     (unless (if (string-prefix? "/" old)
                                 (file-exists? old)
                                 (with-directory-excursion (dirname new*)
                                   (file-exists? old)))
                       (error (format #f "symlink `~a' points to nonexistent \
file `~a'" new* old))))
                   (symlink old new*))
                 (lambda args
                   ;; When doing 'guix system init' on the current '/', some
                   ;; symlinks may already exists.  Override them.
                   (if (= EEXIST (system-error-errno args))
                       (begin
                         (delete-file new*)
                         (try))
                       (apply throw args)))))))))
      (lambda args
        ;; Usually we can only get here when installing to an existing root,
        ;; as with 'guix system init foo.scm /'.
        (format (current-error-port)
                "error: failed to evaluate directive: ~s~%"
                directive)
        (apply throw args)))))

(define (directives store)
  "Return a list of directives to populate the root file system that will host
STORE."
  `((directory ,store 0 0 #o1775)

    (directory "/etc")
    (directory "/var/log")                          ; for shepherd
    (directory "/var/guix/gcroots")
    (directory "/var/empty")                        ; for no-login accounts
    (directory "/var/db")                           ; for dhclient, etc.
    (directory "/mnt")
    (directory "/var/guix/profiles/per-user/root" 0 0)

    ;; Link to the initial system generation.
    ("/var/guix/profiles/system" -> "system-1-link")

    ("/var/guix/gcroots/booted-system" -> "/run/booted-system")
    ("/var/guix/gcroots/current-system" -> "/run/current-system")
    ("/var/guix/gcroots/profiles" -> "/var/guix/profiles")

    (directory "/bin")
    (directory "/tmp" 0 0 #o1777)                 ; sticky bit
    (directory "/var/tmp" 0 0 #o1777)
    (directory "/var/lock" 0 0 #o1777)

    (directory "/home" 0 0)))

(define* (populate-root-file-system system target
                                    #:key (extras '()))
  "Make the essential non-store files and directories on TARGET.  This
includes /etc, /var, /run, /bin/sh, etc., and all the symlinks to SYSTEM.
EXTRAS is a list of directives appended to the built-in directives to populate
TARGET."
  ;; It's expected that some symbolic link targets do not exist yet, so do not
  ;; error on dangling links.
  (for-each (cut evaluate-populate-directive <> target
                 #:error-on-dangling-symlink? #f)
            (append (directives (%store-directory)) extras))

  ;; Add system generation 1.
  (let ((generation-1 (string-append target
                                     "/var/guix/profiles/system-1-link")))
    (let try ()
      (catch 'system-error
        (lambda ()
          (symlink system generation-1))
        (lambda args
          ;; If GENERATION-1 already exists, overwrite it.
          (if (= EEXIST (system-error-errno args))
              (begin
                (delete-file generation-1)
                (try))
              (apply throw args)))))))

(define %root-profile
  "/var/guix/profiles/per-user/root")

(define* (install-database-and-gc-roots root database profile
                                        #:key (profile-name "guix-profile"))
  "Install DATABASE, the store database, under directory ROOT.  Create
PROFILE-NAME and have it link to PROFILE, a store item."
  (define (scope file)
    (string-append root "/" file))

  (define (mkdir-p* dir)
    (mkdir-p (scope dir)))

  (define (symlink* old new)
    (symlink old (scope new)))

  (install-file database (scope "/var/guix/db/"))
  (chmod (scope "/var/guix/db/db.sqlite") #o644)
  (mkdir-p* "/var/guix/profiles")
  (mkdir-p* "/var/guix/gcroots")
  (symlink* "/var/guix/profiles" "/var/guix/gcroots/profiles")

  ;; Make root's profile, which makes it a GC root.
  (mkdir-p* %root-profile)
  (symlink* profile
            (string-append %root-profile "/" profile-name "-1-link"))
  (symlink* (string-append profile-name "-1-link")
            (string-append %root-profile "/" profile-name)))

(define* (populate-single-profile-directory directory
                                            #:key profile closure
                                            (profile-name "guix-profile")
                                            database)
  "Populate DIRECTORY with a store containing PROFILE, whose closure is given
in the file called CLOSURE (as generated by #:references-graphs.)  DIRECTORY
is initialized to contain a single profile under /root pointing to PROFILE.

When DATABASE is true, copy it to DIRECTORY/var/guix/db and create
DIRECTORY/var/guix/gcroots and friends.

PROFILE-NAME is the name of the profile being created under
/var/guix/profiles, typically either \"guix-profile\" or \"current-guix\".

This is used to create the self-contained tarballs with 'guix pack'."
  (define (scope file)
    (string-append directory "/" file))

  (define (mkdir-p* dir)
    (mkdir-p (scope dir)))

  (define (symlink* old new)
    (symlink old (scope new)))

  ;; Populate the store.
  (populate-store (list closure) directory
                  #:deduplicate? #f)

  (when database
    (install-database-and-gc-roots directory database profile
                                   #:profile-name profile-name))

  (match profile-name
    ("guix-profile"
     (mkdir-p* "/root")
     (symlink* (string-append %root-profile "/guix-profile")
               "/root/.guix-profile"))
    ("current-guix"
     (mkdir-p* "/root/.config/guix")
     (symlink* (string-append %root-profile "/current-guix")
               "/root/.config/guix/current"))
    (_
     #t)))

(define (mount-cow-store target backing-directory)
  "Make the store copy-on-write, using TARGET as the backing store.  This is
useful when TARGET is on a hard disk, whereas the current store is on a RAM
disk."
  (define (set-store-permissions directory)
    "Set the right perms on DIRECTORY to use it as the store."
    (chown directory 0 30000)      ;use the fixed 'guixbuild' GID
    (chmod directory #o1775))

  (let ((tmpdir (string-append target "/tmp")))
    (mkdir-p tmpdir)
    (mount tmpdir "/tmp" "none" MS_BIND))

  (let* ((rw-dir (string-append target backing-directory))
         (work-dir (string-append rw-dir "/../.overlayfs-workdir")))
    (mkdir-p rw-dir)
    (mkdir-p work-dir)
    (mkdir-p "/.rw-store")
    (set-store-permissions rw-dir)
    (set-store-permissions "/.rw-store")

    ;; Mount the overlay, then atomically make it the store.
    (mount "none" "/.rw-store" "overlay" 0
           (string-append "lowerdir=" (%store-directory) ","
                          "upperdir=" rw-dir ","
                          "workdir=" work-dir))
    (mount "/.rw-store" (%store-directory) "" MS_MOVE)
    (rmdir "/.rw-store")))

(define (umount* directory)
  "Unmount DIRECTORY, but retry a few times upon EBUSY."
  (let loop ((attempts 5))
    (catch 'system-error
      (lambda ()
        (umount directory))
      (lambda args
        (if (and (= EBUSY (system-error-errno args))
                 (> attempts 0))
            (begin
              (sleep 1)
              (loop (- attempts 1)))
            (apply throw args))))))

(define (unmount-cow-store target backing-directory)
  "Unmount copy-on-write store."
  (let ((tmp-dir "/remove"))
    (mkdir-p tmp-dir)
    (mount (%store-directory) tmp-dir "" MS_MOVE)

    ;; We might get EBUSY at this point, possibly because of lingering
    ;; processes with open file descriptors.  Use 'umount*' to retry upon
    ;; EBUSY, leaving a bit of time.  See <https://issues.guix.gnu.org/59884>.
    (umount* tmp-dir)

    (rmdir tmp-dir)
    (delete-file-recursively
     (string-append target backing-directory))))

;;; install.scm ends here
um (versions 90+, older might also work but are untested), Firefox (versions 60+) and their derivatives.

Building

There're currently 2 ways to build Haketilo.

1. Simple stupid way - build.sh script

You only need a POSIX-compliant environment for this (shell, awk, etc.). It is a viable option if you don't need to run the automated test suite. From project's root directory, using a POSIX shell, you type either:

./build.sh mozilla # to build for Firefox-based browsers

or:

./build.sh chromium # to build for Chromium-based browsers

The unpacked extension shall be generated under ./mozilla-unpacked/ or ./chromium-unpacked/, respectively. You can then load it into your browser as a temporary extension or pack it into an xpi/crx/zip archive manually, e.g.:

7z a -tzip haketilo.xpi -w mozilla-unpacked/.

2. configure-based build

This method assumes you have not only a POSIX environment but also a working Make tool and the zip command. From project's root directory, run the shell commands:

./configure --host=mozilla # or analogically with --host=chromium
make

This would generate the unpacked extension under ./mozilla-unpacked/ and its zipped version under ./mozilla_build.zip (which you can rename to .xpi if you want).

You can also perform an out-of-source build, for example:

mkdir /tmp/haketilo-build && cd /tmp/haketilo-build
/path/to/haketilo/sources/configure --host=chromium
make all # will generate both ./mozilla-build.zip and ./chromium-build.zip

Testing

Requirements

Configuring

Note: like building, testing can be performed out-of-source; this can be useful when testing under multiple browsers simultaneously

Running tests requires you to pass some additional information to configure. Relevant options are:

option name explanation
BROWSER_BINARY Path to the browser's binary executable. Under many scenarios the browser executable in PATH is a shell wrapper around the actual binary. Selenium will refuse to work with that and instead requires the binary to be passed (e.g. /usr/lib/abrowser/abrowser instead of /usr/bin/abrowser
CLEAN_PROFILE Path to a directory with browser profile that is "clean", i.e. has all extensions disabled. This is to mitigate the fact that some browsers pick up globally-installed extensions when creating a new profile for testing and these could interfere with our automated tests. This option can currently be skipped because all tests written so far run the browser in safe mode.
DRIVER Selenium driver command (e.g. geckodriver).
PYTHON Python 3 command. The interpreter spawned with this command is expected to have Pytest and Selenium in its import path.

Options can be specified to configure using the following notations (can be mixed):

./configure --host=mozilla --browser-binary=/usr/local/lib/icecat/icecat
./configure --host mozilla --browser-binary /usr/local/lib/icecat/icecat
./configure HOST=mozilla BROWSER_BINARY=/usr/local/lib/icecat/icecat

configure will try to guess the proper values for options that were not given. To help with this, you can (in some cases) give your actual browser name to --host, for example:

$ ./configure --host=abrowser
Guessing SRCDIR: /home/urz/haketilo-development/browser-extension
Guessing BROWSER_BINARY: /usr/lib/abrowser/abrowser
Guessing DRIVER: /usr/bin/geckodriver
Guessing PYTHON: /usr/bin/python3
Guessing DESTDIR: /usr/share/mozilla/extensions/
$

This may or may not work, depending on the underlying operating system and how the tools were installed.

Running

After configuring, the entire test suite can be run with:

make test

Alternatively, it is possible to run Pytest directly to have some fine-grained control over which tests are run, e.g.:

# Generate the necessary certificates and pytest.ini. This is done automatically
# when running `make test`.
make test-prepare

# Optionally prevent Python from clobbering source directories with .pyc files.
# `make test` rule does the same.
#export PYTHONPYCACHEPREFIX="$(pwd)/test__pycache__"

# Optionally stop Firefox from spawning window(s) during test.
#export MOZ_HEADLESS=whatever

# Run all popup tests with high verbosity.
python3 -m pytest -vv -k popup

As of Haketilo 2.0 some tests may spuriously fail. This is the result of it being notoriously difficult to avoid some weirdnesses when driving Firefox using Selenium. To make sure a failed test is not the result of some more serious bug, you might want to rerun the test suite.

Setting up an environment for manual testing

The automated tests are run with browser's all network requests going through a Python proxy. The proxy allows us to mock websites that are then navigated to in the browser. At times you might want to replicate this test environment while playing manually with the browser. A poor man's approach is to add something like:

from time import sleep
sleep(100000)

inside one of the test functions and then run that test function from Pytest (with MOZ_HEADLESS unset!). This might make sense when debugging some particular test case. For general experiments we have instead provided convenience targets make test-environment and make test-environment-with-haketilo. Running any of those will spawn a browser window together with a Python shell where driver variable will hold Selenium driver object controlling that browser. In case of the test-environment-with-haketilo target the browser will additionally appear with Haketilo loaded into it.

Copying

All copyright information is gathered in the copyright file which follows the format of debian/copyright file. License notices are also present in all text files of the extension.

In general, this entire extension is available under the terms of GPLv3+ with various additional licenses and permissions for particular files.

I, Wojtek Kosior, thereby promise not to sue for violation of this program's licenses. Although I request that you do not make use of this code in a proprietary program, I am not going to enforce this in court.

More documentation

See our wiki for information.

Contributing

Development happens on our Redmine instance.

Alternatively, you can write to koszko@koszko.org.