aboutsummaryrefslogtreecommitdiff
path: root/tests/store-deduplication.scm
blob: e2870a363dcffd76cfa822044cf7e9a4726e461a (about) (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2018 Ludovic Courtès <ludo@gnu.org>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.

(define-module (test-store-deduplication)
  #:use-module (guix tests)
  #:use-module (guix store deduplication)
  #:use-module (gcrypt hash)
  #:use-module ((guix utils) #:select (call-with-temporary-directory))
  #:use-module (guix build utils)
  #:use-module (rnrs bytevectors)
  #:use-module (ice-9 binary-ports)
  #:use-module (srfi srfi-1)
  #:use-module (srfi srfi-64))

(test-begin "store-deduplication")

(test-equal "deduplicate"
  (cons* #t #f                                    ;inode comparisons
         2 (make-list 5 6))                       ;'nlink' values

  (call-with-temporary-directory
   (lambda (store)
     (let ((data      (string->utf8 "Hello, world!"))
           (identical (map (lambda (n)
                             (string-append store "/" (number->string n)
                                            "/a/b/c"))
                           (iota 5)))
           (unique    (string-append store "/unique")))
       (for-each (lambda (file)
                   (mkdir-p (dirname file))
                   (call-with-output-file file
                     (lambda (port)
                       (put-bytevector port data))))
                 identical)
       ;; Make the parent of IDENTICAL read-only.  This should not prevent
       ;; deduplication from inserting its hard link.
       (chmod (dirname (second identical)) #o544)

       (call-with-output-file unique
         (lambda (port)
           (put-bytevector port (string->utf8 "This is unique."))))

       (deduplicate store (nar-sha256 store) #:store store)

       ;; (system (string-append "ls -lRia " store))
       (cons* (apply = (map (compose stat:ino stat) identical))
              (= (stat:ino (stat unique))
                 (stat:ino (stat (car identical))))
              (stat:nlink (stat unique))
              (map (compose stat:nlink stat) identical))))))

(test-equal "deduplicate, ENOSPC"
  (cons* #f                                       ;inode comparison
         (append (make-list 3 4)
                 (make-list 7 1)))                ;'nlink' values

  ;; In this scenario the first 3 files are properly deduplicated and then we
  ;; simulate a full '.links' directory where link(2) gets ENOSPC, thereby
  ;; preventing deduplication of the subsequent files.
  (call-with-temporary-directory
   (lambda (store)
     (let ((true-link link)
           (links     0)
           (data1     (string->utf8 "Hello, world!"))
           (data2     (string->utf8 "Hi, world!"))
           (identical (map (lambda (n)
                             (string-append store "/" (number->string n)
                                            "/a/b/c"))
                           (iota 10)))
           (populate  (lambda (data)
                        (lambda (file)
                          (mkdir-p (dirname file))
                          (call-with-output-file file
                            (lambda (port)
                              (put-bytevector port data)))))))
       (for-each (populate data1) (take identical 5))
       (for-each (populate data2) (drop identical 5))
       (dynamic-wind
         (lambda ()
           (set! link (lambda (old new)
                        (set! links (+ links 1))
                        (if (<= links 3)
                            (true-link old new)
                            (throw 'system-error "link" "~A" '("Whaaat?!")
                                   (list ENOSPC))))))
         (lambda ()
           (deduplicate store (nar-sha256 store) #:store store))
         (lambda ()
           (set! link true-link)))

       (cons (apply = (map (compose stat:ino stat) identical))
             (map (compose stat:nlink stat) identical))))))

(test-end "store-deduplication")
ehe <othacehe@gnu.org> Mathieu Othacehe 2020-09-29images: hurd: Add hurd-barebones-qcow2-image....* gnu/system/images/hurd.scm (hurd-barebones-qcow2-image): New variable. Mathieu Othacehe 2020-09-29image: Add support for compressed-qcow2 format....* gnu/build/image.scm (convert-disk-image): New procedure. (genimage): Remove target argument. * gnu/system/image.scm (system-disk-image): Add support for 'compressed-qcow2 image format. Call "convert-disk-image" to apply image conversions on the final image. Add "qemu-minimal" to the build inputs. (system-image): Also add support for 'compressed-qcow2. Mathieu Othacehe 2020-09-19system: image: Rename ISO9660 files....* gnu/system/image.scm (system-iso9660-image): Change the default file output name to "image.iso". Mathieu Othacehe 2020-09-02install: Factorize cow-store procedure....Move the cow-store procedure from the service declaration in (gnu system install) to (gnu build install), so that it can be called from within a different context than Shepherd. * gnu/build/install.scm (mount-cow-store, unmount-cow-store): New procedures. * gnu/system/install.scm (make-cow-store): Remove it, (cow-store-service-type): adapt it accordingly. Mathieu Othacehe 2020-09-01Revert "system: image: Do not offload image files."...This reverts commit 6a9581741e4ee81226aeb2f1c997df76670a6aab, which is obsoleted by the previous commit. Maxim Cournoyer 2020-08-31guix: system: Add `--label' option....* guix/scripts/system.scm (%options): Add `--label'. (system-derivation-for-action): Take a #:label key to set volume ID. (perform-action): Take a #:label key. (%default-options): Add default label value. (process-action): Pass label value from command-line to perform-action. * gnu/system/image.scm (image-with-label): New procedure. Julien Lepiller 2020-08-31vm: Disable caching for writable file system mappings....Fixes <https://bugs.gnu.org/43062>. Reported by elaexuotee@wilsonb.com. * gnu/system/vm.scm (mapping->file-system)[options]: Disable loose caching when WRITABLE? is true. Ludovic Courtès 2020-08-28system: Remove extra line from 'lightweight-desktop.tmpl'....This is a followup to c7af96871737ffa0e2be879e200573bc60603c8c. * gnu/system/examples/lightweight-desktop.tmpl: Remove extra 'use-package-modules' line. Ludovic Courtès 2020-08-26system: image: Do not offload image files....Image files can be very large (multiple gigabytes), and the process of generating them is primarily I/O bound. As disk access is typically faster than network access, it makes sense build them locally. * gnu/system/image.scm (system-disk-image): Pass the #:local-build? #t parameter to computed-file calls dealing with generating image files. (system-iso9660-image): Likewise. Maxim Cournoyer 2020-08-25Remove "guile-zlib" extension when unused....This is a follow-up of 755f365b02b42a5d1e8ef3000dadef069553a478. As (zlib) is autoloaded in (gnu build linux-modules), "guile-zlib" is needed as an extension only when it is effectively used. * gnu/installer.scm (installer-program): Remove "guile-zlib" from the extensions. * gnu/machine/ssh.scm (machine-check-initrd-modules): Ditto. * gnu/services.scm (activation-script): Ditto. * gnu/services/base.scm (default-serial-port): Ditto, (agetty-shepherd-service): ditto, (udev-service-type): ditto. * gnu/system/image.scm (gcrypt-sqlite3&co): Ditto. * gnu/system/shadow.scm (account-shepherd-service): Ditto. Mathieu Othacehe 2020-08-25linux-libre: Support module compression....This commit adds support for GZIP compression for linux-libre kernel modules. The initrd modules are kept uncompressed as the initrd is already compressed as a whole. The linux-libre kernel also supports XZ compression, but as Guix does not have any available bindings for now, and the compression time is far more significant, GZIP seems to be a better option. * gnu/build/linux-modules.scm (modinfo-section-contents): Use 'call-with-gzip-input-port' to read from a module file using '.gz' extension, (strip-extension): new procedure, (dot-ko): adapt to support compression, (ensure-dot-ko): ditto, (file-name->module-name): ditto, (find-module-file): ditto, (load-linux-module*): ditto, (module-name->file-name/guess): ditto, (module-name-lookup): ditto, (write-module-name-database): ditto, (write-module-alias-database): ditto, (write-module-device-database): ditto. * gnu/installer.scm (installer-program): Add "guile-zlib" to the extensions. * gnu/machine/ssh.scm (machine-check-initrd-modules): Ditto. * gnu/services.scm (activation-script): Ditto. * gnu/services/base.scm (default-serial-port): Ditto, (agetty-shepherd-service): ditto, (udev-service-type): ditto. * gnu/system/image.scm (gcrypt-sqlite3&co): Ditto. * gnu/system/linux-initrd.scm (flat-linux-module-directory): Add "guile-zlib" to the extensions and make sure that the initrd only contains uncompressed module files. * gnu/system/shadow.scm (account-shepherd-service): Add "guile-zlib" to the extensions. * guix/profiles.scm (linux-module-database): Ditto. Mathieu Othacehe 2020-08-03file-systems: Leave room for extension in serialized specs....* gnu/system/file-systems.scm (spec->file-system): Ignore extra fields. (file-system->spec): Add comment. Ludovic Courtès 2020-07-31file-system: Add efivarfs support....Tools such as efibootmgr rely on the deprecated /sys/firmware/efi/vars API as well as on the new /sys/firmware/efi/efivars API. The latter needs to be mounted. Reported by Keyhenge here: https://lists.gnu.org/archive/html/bug-guix/2020-04/msg00274.html Here is the efivarfs documentation: https://www.kernel.org/doc/Documentation/filesystems/efivarfs.txt. * gnu/system/file-systems.scm (%efivars-file-system): New exported variable, (%base-file-systems): add it. * gnu/system/install.scm (%efivars-file-system): Add it. Signed-off-by: Mathieu Othacehe <othacehe@gnu.org> Mathieu Othacehe 2020-07-31file-system: Add mount-may-fail? option....* gnu/system/file-systems.scm (<file-system>): Add a mount-may-fail? field. (file-system->spec): adapt accordingly, (spec->file-system): ditto. * gnu/build/file-systems.scm (mount-file-system): If 'system-error is raised and mount-may-fail? is true, ignore it. Otherwise, re-raise the exception. Signed-off-by: Mathieu Othacehe <othacehe@gnu.org> Mathieu Othacehe 2020-07-31file-systems: Add %debug-file-system....* gnu/system/file-systems.scm (%debug-file-system): New variable, (%base-file-systems): add it. Mathieu Othacehe 2020-07-29examples: Add password for Alice's brother....* gnu/system/examples/desktop.tmpl (users)[bob]: Add password. Oleg Pykhalov 2020-07-26file-systems: Add NTFS support....* gnu/system/uuid.scm (%ntfs-endianness): New macro, (ntfs-uuid->string): new procedure, (%ntfs-endianness): new variable, (string->ntfs-uuid): new exported procedure, (%uuid-parsers): add NTFS support, (%uuid-printers): add NTFS support. * gnu/build/file-systems.scm (%ntfs-endianness): New macro, (ntfs-superblock?, read-ntfs-superblock, ntfs-superblock-uuid, check-ntfs-file-system): new procedure, (%partition-uuid-readers): add NTFS support, (check-file-system): add NTFS support. Mathieu Othacehe 2020-07-25Use 'formatted-message' instead of '&message' where appropriate....* gnu.scm (%try-use-modules): Use 'formatted-message' instead of '&message'. * gnu/machine/digital-ocean.scm (maybe-raise-unsupported-configuration-error): Likewise. * gnu/machine/ssh.scm (machine-check-file-system-availability): Likewise. (machine-check-building-for-appropriate-system): Likewise. (deploy-managed-host): Likewise. (maybe-raise-unsupported-configuration-error): Likewise. * gnu/packages.scm (search-patch): Likewise. * gnu/services.scm (%service-with-default-value): Likewise. (files->etc-directory): Likewise. (fold-services): Likewise. * gnu/system.scm (locale-name->definition*): Likewise. * gnu/system/mapped-devices.scm (check-device-initrd-modules): Likewise. (check-luks-device): Likewise. * guix/channels.scm (latest-channel-instance): Likewise. * guix/cve.scm (json->cve-items): Likewise. * guix/git-authenticate.scm (commit-signing-key): Likewise. (commit-authorized-keys): Likewise. (authenticate-commit): Likewise. (verify-introductory-commit): Likewise. * guix/remote.scm (remote-pipe-for-gexp): Likewise. * guix/scripts/graph.scm (assert-package): Likewise. * guix/scripts/offload.scm (private-key-from-file*): Likewise. * guix/ssh.scm (authenticate-server*): Likewise. (open-ssh-session): Likewise. (remote-inferior): Likewise. * guix/ui.scm (matching-generations): Likewise. * guix/upstream.scm (package-update): Likewise. * tests/channels.scm ("latest-channel-instances, missing introduction for 'guix'"): Catch 'formatted-message?'. ("authenticate-channel, wrong first commit signer"): Likewise. * tests/lint.scm ("patches: not found"): Adjust message string. * tests/packages.scm ("patch not found yields a run-time error"): Catch 'formatted-message?'. * guix/lint.scm (check-patch-file-names): Handle 'formatted-message?'. (check-derivation): Ditto. Ludovic Courtès 2020-07-25file-systems: Convey hint via '&fix-hint'....* gnu/system/file-systems.scm (btrfs-store-subvolume-file-name): Use '&fix-hint' for the hint. Ludovic Courtès 2020-07-25utils: Move '&fix-hint' to (guix diagnostics)....* guix/utils.scm (&fix-hint): Move to... * guix/diagnostics.scm (&fix-hint): ... here. * gnu.scm: Adjust imports accordingly. * gnu/system/mapped-devices.scm: Likewise. * guix/channels.scm: Likewise. * guix/profiles.scm: Likewise. * guix/scripts/system/reconfigure.scm: Likewise. * guix/ssh.scm: Likewise. Ludovic Courtès