;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2016 Ludovic Courtès <ludo@gnu.org>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.

(define-module (run-system-tests)
  #:use-module (gnu tests)
  #:use-module (guix store)
  #:use-module (guix monads)
  #:use-module (guix derivations)
  #:use-module (guix ui)
  #:use-module (srfi srfi-1)
  #:use-module (srfi srfi-34)
  #:use-module (ice-9 match)
  #:export (run-system-tests))

(define (built-derivations* drv)
  (lambda (store)
    (guard (c ((nix-protocol-error? c)
               (values #f store)))
      (values (build-derivations store drv) store))))

(define (filterm mproc lst)                       ;XXX: move to (guix monads)
  (with-monad %store-monad
    (>>= (foldm %store-monad
                (lambda (item result)
                  (mlet %store-monad ((keep? (mproc item)))
                    (return (if keep?
                                (cons item result)
                                result))))
                '()
                lst)
         (lift1 reverse %store-monad))))

(define (run-system-tests . args)
  (define tests
    ;; Honor the 'TESTS' environment variable so that one can select a subset
    ;; of tests to run in the usual way:
    ;;
    ;;   make check-system TESTS=installed-os
    (match (getenv "TESTS")
      (#f
       (all-system-tests))
      ((= string-tokenize (tests ...))
       (filter (lambda (test)
                 (member (system-test-name test) tests))
               (all-system-tests)))))

  (format (current-error-port) "Running ~a system tests...~%"
          (length tests))

  (with-store store
    (run-with-store store
      (mlet* %store-monad ((drv (mapm %store-monad system-test-value tests))
                           (out -> (map derivation->output-path drv)))
        (mbegin %store-monad
          (show-what-to-build* drv)
          (set-build-options* #:keep-going? #t #:keep-failed? #t
                              #:print-build-trace #t
                              #:fallback? #t)
          (built-derivations* drv)
          (mlet %store-monad ((valid  (filterm (store-lift valid-path?)
                                               out))
                              (failed (filterm (store-lift
                                                (negate valid-path?))
                                               out)))
            (format #t "TOTAL: ~a\n" (length drv))
            (for-each (lambda (item)
                        (format #t "PASS: ~a~%" item))
                      valid)
            (for-each (lambda (item)
                        (format #t "FAIL: ~a~%" item))
                      failed)
            (exit (null? failed))))))))
ite): Document it.
</span></span></td><td>Maxim Cournoyer</td></tr>
<tr><td><span title='2021-01-31 22:28:46 -0500'>2021-01-31</span></td><td><a href='/guix/commit/build-aux/test-driver.scm?id=26a66d0feaab7626c4053071d5557394931fe14b'>build: test-driver.scm: Allow running as a standalone script.</a><span class='msg-avail'>...<span class='msg-tooltip'>* build-aux/test-driver.scm: Add an exec-based shebang and set the script
executable bit.
(main): Insert a newline after the version string is printed with --version.
</span></span></td><td>Maxim Cournoyer</td></tr>
<tr><td><span title='2021-01-31 22:28:46 -0500'>2021-01-31</span></td><td><a href='/guix/commit/build-aux/test-driver.scm?id=93a628c4e4f4a7f6665190f3c52a417daebaf28a'>build: test-driver.scm: Add a new '--errors-only' option.</a><span class='msg-avail'>...<span class='msg-tooltip'>* build-aux/test-driver.scm (show-help): Add the help text for the
new '--errors-only' option.
(%options): Add the errors-only option.
(test-runner-gnu): Add the errors-only? parameter and update doc.  Move the
logging of the test data after the test has completed, so a choice can be made
whether to keep it or discard it based on the value of the test result.
(main): Pass the errors-only? option to the driver.
* doc/guix.texi (Running the Test Suite): Document the new option.
</span></span></td><td>Maxim Cournoyer</td></tr>
<tr><td><span title='2021-01-31 22:28:37 -0500'>2021-01-31</span></td><td><a href='/guix/commit/build-aux/test-driver.scm?id=a1ea2acb376c8a74f41ce0e683680417207b365a'>build: test-driver.scm: Add test cases filtering options.</a><span class='msg-avail'>...<span class='msg-tooltip'>* build-aux/test-driver.scm (show-help): Add help text for the new --select
and --exclude options.
(%options): Add the new select and exclude options.
(test-runner-gnu): Pass them to the test runner.  Update doc.
(test-match-name*, test-match-name*/negated, %test-match-all): New variables.
(main): Compute the test specifier based on the values of the new options and
apply it to the current test runner when running the test file.
* doc/guix.texi (Running the Test Suite): Document the new options.
</span></span></td><td>Maxim Cournoyer</td></tr>
<tr><td><span title='2021-01-31 22:23:41 -0500'>2021-01-31</span></td><td><a href='/guix/commit/build-aux/test-driver.scm?id=346210b1b2130112dd5f894c1d40afb37c1c51a5'>build: test-driver.scm: Enable colored test results by default.</a><span class='msg-avail'>...<span class='msg-tooltip'>The Automake parallel test harness does its own smart detection of the
terminal color capability and always provides the --color-tests argument to
the driver.  This change defaults the --color-tests argument to true when the
test driver is run on its own (not via Automake).

* build-aux/test-driver.scm (main): Set the default value of the --color-tests
argument to true when it's not explicitly provided.
</span></span></td><td>Maxim Cournoyer</td></tr>
<tr><td><span title='2021-01-31 22:23:41 -0500'>2021-01-31</span></td><td><a href='/guix/commit/build-aux/test-driver.scm?id=13f299b2c98cce0ede3a8a37dd11832fdb3827bb'>build: test-driver.scm: Make output redirection optional.</a><span class='msg-avail'>...<span class='msg-tooltip'>This makes it easier (and less surprising) for users to experiment with the
custom Scheme test driver directly.  The behavior is unchanged from Automake's
point of view.

* build-aux/test-driver.scm (main): Make the --log-file and --trs-file
arguments optional and update doc.  Only open, redirect and close a port to a log file when
the --log-file option is provided.  Only open and close a port to a trs file
when the --trs-file option is provided.
(test-runner-gnu): Set OUT-PORT parameter default value to the current output
port.  Set the TRS-PORT parameter default value to a void port.  Update doc.
</span></span></td><td>Maxim Cournoyer</td></tr>