aboutsummaryrefslogtreecommitdiff
#define _XOPEN_SOURCE 600

#include "config.h"

#include <cerrno>
#include <algorithm>
#include <vector>
#include <map>

#include <strings.h> // for strcasecmp

#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <dirent.h>
#include <fcntl.h>

#include "archive.hh"
#include "util.hh"


namespace nix {

static string archiveVersion1 = "nix-archive-1";

static string caseHackSuffix = "~nix~case~hack~";

PathFilter defaultPathFilter;


static void dumpContents(const Path & path, size_t size,
    Sink & sink)
{
    writeString("contents", sink);
    writeLongLong(size, sink);

    AutoCloseFD fd = open(path.c_str(), O_RDONLY);
    if (fd == -1) throw SysError(format("opening file `%1%'") % path);

    unsigned char buf[65536];
    size_t left = size;

    while (left > 0) {
        size_t n = left > sizeof(buf) ? sizeof(buf) : left;
        readFull(fd, buf, n);
        left -= n;
        sink(buf, n);
    }

    writePadding(size, sink);
}


static void dump(const Path & path, Sink & sink, PathFilter & filter)
{
    struct stat st;
    if (lstat(path.c_str(), &st))
        throw SysError(format("getting attributes of path `%1%'") % path);

    writeString("(", sink);

    if (S_ISREG(st.st_mode)) {
        writeString("type", sink);
        writeString("regular", sink);
        if (st.st_mode & S_IXUSR) {
            writeString("executable", sink);
            writeString("", sink);
        }
        dumpContents(path, (size_t) st.st_size, sink);
    }

    else if (S_ISDIR(st.st_mode)) {
        writeString("type", sink);
        writeString("directory", sink);

        /* If we're on a case-insensitive system like Mac OS X, undo
           the case hack applied by restorePath(). */
        std::map<string, string> unhacked;
        for (auto & i : readDirectory(path))
	    unhacked[i.name] = i.name;

        for (auto & i : unhacked)
            if (filter(path + "/" + i.first)) {
                writeString("entry", sink);
                writeString("(", sink);
                writeString("name", sink);
                writeString(i.first, sink);
                writeString("node", sink);
                dump(path + "/" + i.second, sink, filter);
                writeString(")", sink);
            }
    }

    else if (S_ISLNK(st.st_mode)) {
        writeString("type", sink);
        writeString("symlink", sink);
        writeString("target", sink);
        writeString(readLink(path), sink);
    }

    else throw Error(format("file `%1%' has an unsupported type") % path);

    writeString(")", sink);
}


void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
{
    writeString(archiveVersion1, sink);
    dump(path, sink, filter);
}


static SerialisationError badArchive(string s)
{
    return SerialisationError("bad archive: " + s);
}


#if 0
static void skipGeneric(Source & source)
{
    if (readString(source) == "(") {
        while (readString(source) != ")")
            skipGeneric(source);
    }
}
#endif


static void parseContents(ParseSink & sink, Source & source, const Path & path)
{
    unsigned long long size = readLongLong(source);

    sink.preallocateContents(size);

    unsigned long long left = size;
    unsigned char buf[65536];

    while (left) {
        checkInterrupt();
        unsigned int n = sizeof(buf);
        if ((unsigned long long) n > left) n = left;
        source(buf, n);
        sink.receiveContents(buf, n);
        left -= n;
    }

    readPadding(size, source);
}


struct CaseInsensitiveCompare
{
    bool operator() (const string & a, const string & b) const
    {
        return strcasecmp(a.c_str(), b.c_str()) < 0;
    }
};


static void parse(ParseSink & sink, Source & source, const Path & path)
{
    string s;

    s = readString(source);
    if (s != "(") throw badArchive("expected open tag");

    enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown;

    std::map<Path, int, CaseInsensitiveCompare> names;

    while (1) {
        checkInterrupt();

        s = readString(source);

        if (s == ")") {
            break;
        }

        else if (s == "type") {
            if (type != tpUnknown)
                throw badArchive("multiple type fields");
            string t = readString(source);

            if (t == "regular") {
                type = tpRegular;
                sink.createRegularFile(path);
            }

            else if (t == "directory") {
                sink.createDirectory(path);
                type = tpDirectory;
            }

            else if (t == "symlink") {
                type = tpSymlink;
            }

            else throw badArchive("unknown file type " + t);

        }

        else if (s == "contents" && type == tpRegular) {
            parseContents(sink, source, path);
        }

        else if (s == "executable" && type == tpRegular) {
            readString(source);
            sink.isExecutable();
        }

        else if (s == "entry" && type == tpDirectory) {
            string name, prevName;

            s = readString(source);
            if (s != "(") throw badArchive("expected open tag");

            while (1) {
                checkInterrupt();

                s = readString(source);

                if (s == ")") {
                    break;
                } else if (s == "name") {
                    name = readString(source);
                    if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos)
                        throw Error(format("NAR contains invalid file name `%1%'") % name);
                    if (name <= prevName)
                        throw Error("NAR directory is not sorted");
                    prevName = name;
                } else if (s == "node") {
                    if (s.empty()) throw badArchive("entry name missing");
                    parse(sink, source, path + "/" + name);
                } else
                    throw badArchive("unknown field " + s);
            }
        }

        else if (s == "target" && type == tpSymlink) {
            string target = readString(source);
            sink.createSymlink(path, target);
        }

        else
            throw badArchive("unknown field " + s);
    }
}


void parseDump(ParseSink & sink, Source & source)
{
    string version;
    try {
        version = readString(source);
    } catch (SerialisationError & e) {
        /* This generally means the integer at the start couldn't be
           decoded.  Ignore and throw the exception below. */
    }
    if (version != archiveVersion1)
        throw badArchive("input doesn't look like a normalized archive");
    parse(sink, source, "");
}


struct RestoreSink : ParseSink
{
    Path dstPath;
    AutoCloseFD fd;

    void createDirectory(const Path & path)
    {
        Path p = dstPath + path;
        if (mkdir(p.c_str(), 0777) == -1)
            throw SysError(format("creating directory `%1%'") % p);
    };

    void createRegularFile(const Path & path)
    {
        Path p = dstPath + path;
        fd.close();
        fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0666);
        if (fd == -1) throw SysError(format("creating file `%1%'") % p);
    }

    void isExecutable()
    {
        struct stat st;
        if (fstat(fd, &st) == -1)
            throw SysError("fstat");
        if (fchmod(fd, st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
            throw SysError("fchmod");
    }

    void preallocateContents(unsigned long long len)
    {
#if HAVE_POSIX_FALLOCATE
        if (len) {
            errno = posix_fallocate(fd, 0, len);
            /* Note that EINVAL may indicate that the underlying
               filesystem doesn't support preallocation (e.g. on
               OpenSolaris).  Since preallocation is just an
               optimisation, ignore it. */
            if (errno && errno != EINVAL)
                throw SysError(format("preallocating file of %1% bytes") % len);
        }
#endif
    }

    void receiveContents(unsigned char * data, unsigned int len)
    {
        writeFull(fd, data, len);
    }

    void createSymlink(const Path & path, const string & target)
    {
        Path p = dstPath + path;
        nix::createSymlink(target, p);
    }
};


void restorePath(const Path & path, Source & source)
{
    RestoreSink sink;
    sink.dstPath = path;
    parseDump(sink, source);
}


}
06-27derivations: <derivation-input> now aggregates a <derivation>....Consequently, the whole graph of <derivation> object is readily available without having to go through 'read-derivation-from-file', which could have cache misses if the requested <derivation> object had been GC'd in the meantime. This is an important property for the performance of things like 'derivation-build-plan' that traverse the derivation graph. * guix/derivations.scm (<derivation-input>): Replace 'path' field by 'derivation'. (derivation-input-path): Adjust accordingly. (derivation-input-key): New procedure. (derivation-input-output-paths): Adjust accordingly. (coalesce-duplicate-inputs): Likewise. (derivation-prerequisites): Use 'derivation-input-key' to compute keys for INPUT-SET. (derivation-build-plan): Likewise. (read-derivation): Add optional 'read-derivation-from-file' parameter. [make-input-drvs]: Call it. (write-derivation)[write-input]: Adjust to new <derivation-input>. (derivation/masked-inputs): Likewise, and remove redundant 'coalesce-duplicate-inputs' call. (derivation)[input->derivation-input]: Change to consider only the derivation case. Update call to 'make-derivation-input'. [input->source]: New procedure. Separate sources from inputs. (map-derivation): Adjust to new <derivation-input>. * tests/derivations.scm ("parse & export"): Pass a second argument to 'read-derivation'. ("build-expression->derivation and derivation-prerequisites") ("derivation-prerequisites and valid-derivation-input?"): Adjust to new <derivation-input>. Ludovic Courtès 2019-06-27derivations: Rewrite and replace 'derivations-prerequisites-to-build'....The new 'derivation-build-plan' procedure has a more appropriate signature: it takes a list of <derivation-inputs> instead of taking one <derivation>. Its body is also much simpler. * guix/derivations.scm (derivation-build-plan): New procedure. (derivation-prerequisites-to-build): Express in terms of 'derivation-build-plan' and mark as deprecated. * tests/derivations.scm: Change 'derivation-prerequisites-to-build' tests to 'derivation-build-plan' and adjust accordingly. Ludovic Courtès 2019-06-20Merge branch 'staging' into core-updatesMarius Bakke 2019-06-18tests: Avoid 'delete-paths' calls for 'derivation-prerequisites-to-build'....* tests/derivations.scm ("derivation-prerequisites-to-build when outputs already present"): Insert random text in BUILDER, and remove two calls to 'delete-paths'. Use 'derivation->output-path' for INPUT-PATH instead of an old idiom. Ludovic Courtès 2019-06-14packages: Remove 'search-bootstrap-binary'....* gnu/packages.scm (%bootstrap-binaries-path, search-bootstrap-binary): Remove. * gnu/packages/bootstrap.scm (bootstrap-executable): Export. * guix/tests.scm (bootstrap-binary-file, search-bootstrap-binary): Export. * tests/derivations.scm: Remove (gnu packages) import. * tests/grafts.scm: Likewise. * tests/guix-daemon.sh: Likewise. Ludovic Courtès 2019-06-10store: 'build-things' accepts derivation/output pairs....This allows callers to request the substitution of a single derivation output. * guix/store.scm (build-things): Accept derivation/output pairs among THINGS. * guix/derivations.scm (build-derivations): Likewise. * tests/store.scm ("substitute + build-things with specific output"): New test. * tests/derivations.scm ("build-derivations with specific output"): New test. * doc/guix.texi (The Store): Adjust accordingly. Ludovic Courtès 2019-02-04daemon: Rename 'NIX_STATE_DIR' and 'NIX_DB_DIR' environment variables....Fixes <https://bugs.gnu.org/22459>. Reported by Jeff Mickey <j@codemac.net>. * guix/config.scm.in (%state-directory): Change NIX_STATE_DIR to GUIX_STATE_DIRECTORY. (%store-database-directory): Change NIX_DB_DIR to GUIX_DATABASE_DIRECTORY. * nix/libstore/globals.cc (Settings::processEnvironment): Likewise. * guix/self.scm (make-config.scm): Likewise. * build-aux/build-self.scm (make-config.scm): Likewise. * build-aux/test-env.in: Likewise. * tests/derivations.scm ("derivation #:leaked-env-vars"): Likewise. * tests/guix-build.sh (GUIX_DAEMON_SOCKET): Likewise. * tests/guix-daemon.sh (socket): Likewise. Ludovic Courtès 2019-01-21store: Rename '&nix-error' to '&store-error'....* guix/store.scm (&nix-error): Rename to... (&store-error): ... this, and adjust users. (&nix-connection-error): Rename to... (&store-connection-error): ... this, and adjust users. (&nix-protocol-error): Rename to... (&store-protocol-error): ... this, adjust users. (&nix-error, &nix-connection-error, &nix-protocol-error): Define these condition types and their getters as deprecrated aliases. * build-aux/run-system-tests.scm, guix/derivations.scm, guix/grafts.scm, guix/scripts/challenge.scm, guix/scripts/graph.scm, guix/scripts/lint.scm, guix/scripts/offload.scm, guix/serialization.scm, guix/ssh.scm, guix/tests.scm, guix/ui.scm, tests/derivations.scm, tests/gexp.scm, tests/guix-daemon.sh, tests/packages.scm, tests/store.scm, doc/guix.texi: Adjust to use the new names. Ludovic Courtès 2018-11-28derivations: Add properties....* guix/derivations.scm (derivation): Add #:properties parameter. [user+system-env-vars]: Honor it. (derivation-properties): New procedure. (build-expression->derivation): Add #:properties and pass it to 'derivation'. * guix/gexp.scm (gexp->derivation): Likewise. * tests/derivations.scm ("derivation-properties"): New test. * tests/gexp.scm ("gexp->derivation properties"): New test. * doc/guix.texi (Derivations, G-Expressions): Adjust accordingly. Ludovic Courtès 2018-09-04Switch to Guile-Gcrypt....This removes (guix hash) and (guix pk-crypto), which now live as part of Guile-Gcrypt (version 0.1.0.) * guix/gcrypt.scm, guix/hash.scm, guix/pk-crypto.scm, tests/hash.scm, tests/pk-crypto.scm: Remove. * configure.ac: Test for Guile-Gcrypt. Remove LIBGCRYPT and LIBGCRYPT_LIBDIR assignments. * m4/guix.m4 (GUIX_ASSERT_LIBGCRYPT_USABLE): Remove. * README: Add Guile-Gcrypt to the dependencies; move libgcrypt as "required unless --disable-daemon". * doc/guix.texi (Requirements): Likewise. * gnu/packages/bash.scm, guix/derivations.scm, guix/docker.scm, guix/git.scm, guix/http-client.scm, guix/import/cpan.scm, guix/import/cran.scm, guix/import/crate.scm, guix/import/elpa.scm, guix/import/gnu.scm, guix/import/hackage.scm, guix/import/texlive.scm, guix/import/utils.scm, guix/nar.scm, guix/pki.scm, guix/scripts/archive.scm, guix/scripts/authenticate.scm, guix/scripts/download.scm, guix/scripts/hash.scm, guix/scripts/pack.scm, guix/scripts/publish.scm, guix/scripts/refresh.scm, guix/scripts/substitute.scm, guix/store.scm, guix/store/deduplication.scm, guix/tests.scm, tests/base32.scm, tests/builders.scm, tests/challenge.scm, tests/cpan.scm, tests/crate.scm, tests/derivations.scm, tests/gem.scm, tests/nar.scm, tests/opam.scm, tests/pki.scm, tests/publish.scm, tests/pypi.scm, tests/store-deduplication.scm, tests/store.scm, tests/substitute.scm: Adjust imports. * gnu/system/vm.scm: Likewise. (guile-sqlite3&co): Rename to... (gcrypt-sqlite3&co): ... this. Add GUILE-GCRYPT. (expression->derivation-in-linux-vm)[config]: Remove. (iso9660-image)[config]: Remove. (qemu-image)[config]: Remove. (system-docker-image)[config]: Remove. * guix/scripts/pack.scm: Adjust imports. (guile-sqlite3&co): Rename to... (gcrypt-sqlite3&co): ... this. Add GUILE-GCRYPT. (self-contained-tarball)[build]: Call 'make-config.scm' without #:libgcrypt argument. (squashfs-image)[libgcrypt]: Remove. [build]: Call 'make-config.scm' without #:libgcrypt. (docker-image)[config, json]: Remove. [build]: Add GUILE-GCRYPT to the extensions Remove (guix config) from the imported modules. * guix/self.scm (specification->package): Remove "libgcrypt", add "guile-gcrypt". (compiled-guix): Remove #:libgcrypt. [guile-gcrypt]: New variable. [dependencies]: Add it. [*core-modules*]: Remove #:libgcrypt from 'make-config.scm' call. Add #:extensions. [*config*]: Remove #:libgcrypt from 'make-config.scm' call. (%dependency-variables): Remove %libgcrypt. (make-config.scm): Remove #:libgcrypt. * build-aux/build-self.scm (guile-gcrypt): New variable. (make-config.scm): Remove #:libgcrypt. (build-program)[fake-gcrypt-hash]: New variable. Add (gcrypt hash) to the imported modules. Adjust load path assignments. * gnu/packages/package-management.scm (guix)[propagated-inputs]: Add GUILE-GCRYPT. [arguments]: In 'wrap-program' phase, add GUILE-GCRYPT to the search path. Ludovic Courtès