aboutsummaryrefslogtreecommitdiff
#pragma once

#include "types.hh"

#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include <unistd.h>
#include <signal.h>
#include <map>
#include <functional>

#include <cstdio>


namespace nix {


#define foreach(it_type, it, collection)                                \
    for (it_type it = (collection).begin(); it != (collection).end(); ++it)

#define foreach_reverse(it_type, it, collection)                                \
    for (it_type it = (collection).rbegin(); it != (collection).rend(); ++it)


/* Return an environment variable. */
string getEnv(const string & key, const string & def = "");

/* Return an absolutized path, resolving paths relative to the
   specified directory, or the current directory otherwise.  The path
   is also canonicalised. */
Path absPath(Path path, Path dir = "");

/* Canonicalise a path by removing all `.' or `..' components and
   double or trailing slashes.  Optionally resolves all symlink
   components such that each component of the resulting path is *not*
   a symbolic link. */
Path canonPath(const Path & path, bool resolveSymlinks = false);

/* Return the directory part of the given canonical path, i.e.,
   everything before the final `/'.  If the path is the root or an
   immediate child thereof (e.g., `/foo'), this means an empty string
   is returned. */
Path dirOf(const Path & path);

/* Return the base name of the given canonical path, i.e., everything
   following the final `/'. */
string baseNameOf(const Path & path);

/* Check whether a given path is a descendant of the given
   directory. */
bool isInDir(const Path & path, const Path & dir);

/* Get status of `path'. */
struct stat lstat(const Path & path);

/* Return true iff the given path exists. */
bool pathExists(const Path & path);

/* Read the contents (target) of a symbolic link.  The result is not
   in any way canonicalised. */
Path readLink(const Path & path);

bool isLink(const Path & path);

/* Read the contents of a directory.  The entries `.' and `..' are
   removed. */
struct DirEntry
{
    string name;
    ino_t ino;
    unsigned char type; // one of DT_*
    DirEntry(const string & name, ino_t ino, unsigned char type)
        : name(name), ino(ino), type(type) { }
};

typedef vector<DirEntry> DirEntries;

DirEntries readDirectory(const Path & path);

unsigned char getFileType(const Path & path);

/* Read the contents of a file into a string. */
string readFile(int fd);
string readFile(const Path & path, bool drain = false);

/* Write a string to a file. */
void writeFile(const Path & path, const string & s);

/* Read a line from a file descriptor. */
string readLine(int fd);

/* Write a line to a file descriptor. */
void writeLine(int fd, string s);

/* Delete a path; i.e., in the case of a directory, it is deleted
   recursively.  Don't use this at home, kids.  The second variant
   returns the number of bytes and blocks freed, and 'linkThreshold' denotes
   the number of links under which a file is accounted for in 'bytesFreed'.  */
void deletePath(const Path & path);

void deletePath(const Path & path, unsigned long long & bytesFreed,
    size_t linkThreshold = 1);

/* Copy SOURCE to DESTINATION, recursively, preserving ownership.  Throw if
   SOURCE contains a file that is not a regular file, symlink, or directory.
   When DELETESOURCE is true, delete source files once they have been
   copied.  */
void copyFileRecursively(const Path &source, const Path &destination,
    bool deleteSource = false);

/* Create a temporary directory. */
Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
    bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);

/* Create a directory and all its parents, if necessary.  Returns the
   list of created directories, in order of creation. */
Paths createDirs(const Path & path);

/* Create a symlink. */
void createSymlink(const Path & target, const Path & link);


template<class T, class A>
T singleton(const A & a)
{
    T t;
    t.insert(a);
    return t;
}


/* Messages. */


typedef enum {
    ltPretty,   /* nice, nested output */
    ltEscapes,  /* nesting indicated using escape codes (for log2xml) */
    ltFlat      /* no nesting */
} LogType;

extern LogType logType;
extern Verbosity verbosity; /* suppress msgs > this */

class Nest
{
private:
    bool nest;
public:
    Nest();
    ~Nest();
    void open(Verbosity level, const FormatOrString & fs);
    void close();
};

void printMsg_(Verbosity level, const FormatOrString & fs);

#define startNest(varName, level, f) \
    Nest varName; \
    if (level <= verbosity) { \
      varName.open(level, (f)); \
    }

#define printMsg(level, f) \
    do { \
        if (level <= nix::verbosity) { \
            nix::printMsg_(level, (f)); \
        } \
    } while (0)

#define debug(f) printMsg(lvlDebug, f)

void warnOnce(bool & haveWarned, const FormatOrString & fs);

void writeToStderr(const string & s);

extern void (*_writeToStderr) (const unsigned char * buf, size_t count);


/* Wrappers arount read()/write() that read/write exactly the
   requested number of bytes. */
void readFull(int fd, unsigned char * buf, size_t count);
void writeFull(int fd, const unsigned char * buf, size_t count);
void writeFull(int fd, const string & s);

MakeError(EndOfFile, Error)


/* Read a file descriptor until EOF occurs. */
string drainFD(int fd);



/* Automatic cleanup of resources. */


template <class T>
struct AutoDeleteArray
{
    T * p;
    AutoDeleteArray(T * p) : p(p) { }
    ~AutoDeleteArray()
    {
        delete [] p;
    }
};


class AutoDelete
{
    Path path;
    bool del;
    bool recursive;
public:
    AutoDelete(const Path & p, bool recursive = true);
    ~AutoDelete();
    void cancel();
};


class AutoCloseFD
{
    int fd;
public:
    AutoCloseFD();
    AutoCloseFD(int fd);
    AutoCloseFD(const AutoCloseFD & fd);
    ~AutoCloseFD();
    void operator =(int fd);
    operator int() const;
    void close();
    bool isOpen();
    int borrow();
};


class Pipe
{
public:
    AutoCloseFD readSide, writeSide;
    void create();
};


class AutoCloseDir
{
    DIR * dir;
public:
    AutoCloseDir();
    AutoCloseDir(DIR * dir);
    ~AutoCloseDir();
    void operator =(DIR * dir);
    operator DIR *();
    void close();
};


class Pid
{
    pid_t pid;
    bool separatePG;
    int killSignal;
public:
    Pid();
    Pid(pid_t pid);
    ~Pid();
    void operator =(pid_t pid);
    operator pid_t();
    void kill(bool quiet = false);
    int wait(bool block);
    void setSeparatePG(bool separatePG);
    void setKillSignal(int signal);
};

/* An "agent" is a helper program that runs in the background and that we talk
   to over pipes, such as the "guix offload" program.  */
struct Agent
{
    /* Pipes for talking to the agent. */
    Pipe toAgent;

    /* Pipe for the agent's standard output/error. */
    Pipe fromAgent;

    /* Pipe for build standard output/error--e.g., for build processes started
       by "guix offload".  */
    Pipe builderOut;

    /* The process ID of the agent. */
    Pid pid;

    /* The command and arguments passed to the agent along with a list of
       environment variable name/value pairs.  */
    Agent(const string &command, const Strings &args,
	  const std::map<string, string> &env = std::map<string, string>());

    ~Agent();
};


/* Kill all processes running under the specified uid by sending them
   a SIGKILL. */
void killUser(uid_t uid);


/* Fork a process that runs the given function, and return the child
   pid to the caller. */
pid_t startProcess(std::function<void()> fun, bool dieWithParent = true,
    const string & errorPrefix = "error: ", bool runExitHandlers = false);


/* Run a program and return its stdout in a string (i.e., like the
   shell backtick operator). */
string runProgram(Path program, bool searchPath = false,
    const Strings & args = Strings());

MakeError(ExecError, Error)

/* Convert a list of strings to a null-terminated vector of char
   *'s. The result must not be accessed beyond the lifetime of the
   list of strings. */
std::vector<char *> stringsToCharPtrs(const Strings & ss);

/* Close all file descriptors except stdin, stdout, stderr, and those
   listed in the given set.  Good practice in child processes. */
void closeMostFDs(const set<int> & exceptions);

/* Set the close-on-exec flag for the given file descriptor. */
void closeOnExec(int fd);

/* Common initialisation performed in child processes. */
void commonChildInit(Pipe & logPipe);

/* User interruption. */

extern volatile sig_atomic_t _isInterrupted;

void _interrupted();

void inline checkInterrupt()
{
    if (_isInterrupted) _interrupted();
}

MakeError(Interrupted, BaseError)


/* String tokenizer. */
template<class C> C tokenizeString(const string & s, const string & separators = " \t\n\r");


/* Concatenate the given strings with a separator between the
   elements. */
string concatStringsSep(const string & sep, const Strings & ss);
string concatStringsSep(const string & sep, const StringSet & ss);


/* Remove trailing whitespace from a string. */
string chomp(const string & s);


/* Convert the exit status of a child as returned by wait() into an
   error string. */
string statusToString(int status);

bool statusOk(int status);


/* Parse a string into an integer. */
template<class N> bool string2Int(const string & s, N & n)
{
    std::istringstream str(s);
    str >> n;
    return str && str.get() == EOF;
}


/* Return true iff `s' ends in `suffix'. */
bool hasSuffix(const string & s, const string & suffix);


/* Read string `s' from stream `str'. */
void expect(std::istream & str, const string & s);

MakeError(FormatError, Error)


/* Read a C-style string from stream `str'. */
string parseString(std::istream & str);


/* Utility function used to parse legacy ATerms. */
bool endOfList(std::istream & str);


/* Exception handling in destructors: print an error message, then
   ignore the exception. */
void ignoreException();


}
#:key name system) "Return the job for IMAGE on SYSTEM. If NAME is passed, use it as job name, otherwise use the IMAGE name." (let* ((image-name (or name (symbol->string (image-name image)))) (name (string-append image-name "." system)) (drv (run-with-store store (mbegin %store-monad (set-guile-for-build (default-guile)) (lower-object (system-image image) system))))) (parameterize ((%graft? #f)) (derivation->job name drv)))) (define* (image-jobs store system #:key source commit) "Return a list of jobs that build images for SYSTEM." (define MiB (expt 2 20)) (parameterize ((current-guix-package (channel-source->package source #:commit commit))) (if (member system %guix-system-supported-systems) `(,(image->job store (image (inherit mbr-hybrid-disk-image) (operating-system installation-os)) #:name "usb-image" #:system system) ,(image->job store (image (inherit (image-with-label iso9660-image (string-append "GUIX_" system "_" (if (> (string-length %guix-version) 7) (substring %guix-version 0 7) %guix-version)))) (operating-system installation-os)) #:name "iso9660-image" #:system system) ;; Only cross-compile Guix System images from x86_64-linux for now. ,@(if (string=? system "x86_64-linux") (map (cut image->job store <> #:system system) %guix-system-images) '())) '()))) (define* (system-test-jobs store system #:key source commit) "Return a list of jobs for the system tests." (define (->job test) (let ((name (string-append "test." (system-test-name test) "." system)) (drv (run-with-store store (mbegin %store-monad (set-current-system system) (set-grafting #f) (set-guile-for-build (default-guile)) (system-test-value test))))) (derivation->job name drv))) (if (member system %guix-system-supported-systems) ;; Override the value of 'current-guix' used by system tests. Using a ;; channel instance makes tests that rely on 'current-guix' less ;; expensive. It also makes sure we get a valid Guix package when this ;; code is not running from a checkout. (parameterize ((current-guix-package (channel-source->package source #:commit commit))) (map ->job (all-system-tests))) '())) (define (tarball-jobs store system) "Return jobs to build the self-contained Guix binary tarball." (define (->job name drv) (let ((name (string-append name "." system))) (parameterize ((%graft? #f)) (derivation->job name drv)))) ;; XXX: Add a job for the stable Guix? (list (->job "binary-tarball" (run-with-store store (mbegin %store-monad (set-guile-for-build (default-guile)) (>>= (profile-derivation (packages->manifest (list guix))) (lambda (profile) (self-contained-tarball "guix-binary" profile #:profile-name "current-guix" #:localstatedir? #t #:compressor (lookup-compressor "xz"))))) #:system system)))) (define job-name ;; Return the name of a package's job. package-name) (define base-packages (mlambda (system) "Return the set of packages considered to be part of the base for SYSTEM." (delete-duplicates (append-map (match-lambda ((_ package _ ...) (match (package-transitive-inputs package) (((_ inputs _ ...) ...) inputs)))) (%final-inputs system))))) (define package->job (lambda* (store package system #:key (suffix "")) "Return a job for PACKAGE on SYSTEM, or #f if this combination is not valid. Append SUFFIX to the job name." (cond ((member package (base-packages system)) (package-job store (string-append "base." (job-name package)) package system #:suffix suffix)) ((supported-package? package system) (let ((drv (package-derivation store package system #:graft? #f))) (and (substitutable-derivation? drv) (package-job store (job-name package) package system #:suffix suffix)))) (else #f)))) (define %x86-64-micro-architectures ;; Micro-architectures for which we build tuned variants. '("haswell" "skylake" "x86-64-v2" "x86-64-v3" "x86-64-v4")) (define (tuned-package-jobs store package system) "Return a list of jobs for PACKAGE tuned for SYSTEM's micro-architectures." (filter-map (lambda (micro-architecture) (define suffix (string-append "." micro-architecture)) (package->job store (tuned-package package micro-architecture) system #:suffix suffix)) (match system ("x86_64-linux" %x86-64-micro-architectures) (_ '())))) (define (all-packages) "Return the list of packages to build." (define (adjust package result) (cond ((package-replacement package) ;; XXX: If PACKAGE and its replacement have the same name/version, ;; then both Cuirass jobs will have the same name, which ;; effectively means that the second one will be ignored. Thus, ;; return the replacement first. (cons* (package-replacement package) ;build both package result)) ((package-superseded package) result) ;don't build it (else (cons package result)))) (fold-packages adjust (fold adjust '() ;include base packages (match (%final-inputs) (((labels packages _ ...) ...) packages))) #:select? (const #t))) ;include hidden packages (define (arguments->manifests arguments channels) "Return the list of manifests extracted from ARGUMENTS." (map (lambda (manifest) (any (lambda (checkout) (let ((path (in-vicinity checkout manifest))) (and (file-exists? path) path))) (map channel-url channels))) arguments)) (define (manifests->jobs store manifests systems) "Return the list of jobs for the entries in MANIFESTS, a list of file names, for each one of SYSTEMS." (define (load-manifest manifest) (save-module-excursion (lambda () (set-current-module (make-user-module '((guix profiles) (gnu)))) (primitive-load manifest)))) (define (manifest-entry-job-name entry) (string-append (manifest-entry-name entry) "-" (manifest-entry-version entry))) (define (manifest-entry->job entry system) (let* ((obj (manifest-entry-item entry)) (drv (parameterize ((%graft? #f)) (run-with-store store (lower-object obj system) #:system system))) (max-silent-time (or (and (package? obj) (assoc-ref (package-properties obj) 'max-silent-time)) 3600)) (timeout (or (and (package? obj) (assoc-ref (package-properties obj) 'timeout)) (* 5 3600)))) (derivation->job (manifest-entry-job-name entry) drv #:max-silent-time max-silent-time #:timeout timeout))) (let ((entries (delete-duplicates (append-map (compose manifest-entries load-manifest) manifests) manifest-entry=?))) (append-map (lambda (system) (map (cut manifest-entry->job <> system) entries)) systems))) (define (arguments->systems arguments) "Return the systems list from ARGUMENTS." (match (assoc-ref arguments 'systems) (#f %cuirass-supported-systems) ((lst ...) lst) ((? string? str) (call-with-input-string str read)))) ;;; ;;; Cuirass entry point. ;;; (define (cuirass-jobs store arguments) "Register Cuirass jobs." (define subset (assoc-ref arguments 'subset)) (define systems (arguments->systems arguments)) (define channels (let ((channels (assq-ref arguments 'channels))) (map sexp->channel channels))) (define guix (find guix-channel? channels)) (define commit (channel-commit guix)) (define source (channel-url guix)) ;; Turn off grafts. Grafting is meant to happen on the user's machines. (parameterize ((%graft? #f)) ;; Return one job for each package, except bootstrap packages. (append-map (lambda (system) (format (current-error-port) "evaluating for '~a' (heap size: ~a MiB)...~%" system (round (/ (assoc-ref (gc-stats) 'heap-size) (expt 2. 20)))) (invalidate-derivation-caches!) (match subset ('all ;; Build everything, including replacements. (let ((all (all-packages)) (jobs (lambda (package) (match (package->job store package system) (#f '()) (main-job (cons main-job (if (tunable-package? package) (tuned-package-jobs store package system) '()))))))) (append (append-map jobs all) (cross-jobs store system)))) ('core ;; Build core packages only. (append (map (lambda (package) (package-job store (job-name package) package system)) (append (commencement-packages system) %core-packages)) (cross-jobs store system))) ('guix ;; Build Guix modules only. (guix-jobs store systems #:source source #:commit commit)) ('hello ;; Build hello package only. (let ((hello (specification->package "hello"))) (list (package-job store (job-name hello) hello system)))) ('images ;; Build Guix System images only. (image-jobs store system #:source source #:commit commit)) ('system-tests ;; Build Guix System tests only. (system-test-jobs store system #:source source #:commit commit)) ('tarball ;; Build Guix tarball only. (tarball-jobs store system)) (('custom . modules) ;; Build custom modules jobs only. (append-map (lambda (module) (let ((proc (module-ref (resolve-interface module) 'cuirass-jobs))) (proc store arguments))) modules)) (('channels . channels) ;; Build only the packages from CHANNELS. (let ((all (all-packages))) (filter-map (lambda (package) (any (lambda (channel) (and (member (channel-name channel) channels) (package->job store package system))) (package-channels package))) all))) (('packages . rest) ;; Build selected list of packages only. (let ((packages (map specification->package rest))) (map (lambda (package) (package-job store (job-name package) package system)) packages))) (('manifests . rest) ;; Build packages in the list of manifests. (let ((manifests (arguments->manifests rest channels))) (manifests->jobs store manifests systems))) (else (error "unknown subset" subset)))) systems)))