diff options
Diffstat (limited to 'third_party/nix/src/libutil')
34 files changed, 7298 insertions, 0 deletions
diff --git a/third_party/nix/src/libutil/affinity.cc b/third_party/nix/src/libutil/affinity.cc new file mode 100644 index 000000000000..98f8287ada67 --- /dev/null +++ b/third_party/nix/src/libutil/affinity.cc @@ -0,0 +1,55 @@ +#include "types.hh" +#include "util.hh" +#include "affinity.hh" + +#if __linux__ +#include <sched.h> +#endif + +namespace nix { + + +#if __linux__ +static bool didSaveAffinity = false; +static cpu_set_t savedAffinity; +#endif + + +void setAffinityTo(int cpu) +{ +#if __linux__ + if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; + didSaveAffinity = true; + debug(format("locking this thread to CPU %1%") % cpu); + cpu_set_t newAffinity; + CPU_ZERO(&newAffinity); + CPU_SET(cpu, &newAffinity); + if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) + printError(format("failed to lock thread to CPU %1%") % cpu); +#endif +} + + +int lockToCurrentCPU() +{ +#if __linux__ + int cpu = sched_getcpu(); + if (cpu != -1) setAffinityTo(cpu); + return cpu; +#else + return -1; +#endif +} + + +void restoreAffinity() +{ +#if __linux__ + if (!didSaveAffinity) return; + if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) + printError("failed to restore affinity %1%"); +#endif +} + + +} diff --git a/third_party/nix/src/libutil/affinity.hh b/third_party/nix/src/libutil/affinity.hh new file mode 100644 index 000000000000..c1bd28e1367a --- /dev/null +++ b/third_party/nix/src/libutil/affinity.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix { + +void setAffinityTo(int cpu); +int lockToCurrentCPU(); +void restoreAffinity(); + +} diff --git a/third_party/nix/src/libutil/archive.cc b/third_party/nix/src/libutil/archive.cc new file mode 100644 index 000000000000..3aa120270970 --- /dev/null +++ b/third_party/nix/src/libutil/archive.cc @@ -0,0 +1,378 @@ +#include <cerrno> +#include <algorithm> +#include <vector> +#include <map> + +#include <strings.h> // for strcasecmp + +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <dirent.h> +#include <fcntl.h> + +#include "archive.hh" +#include "util.hh" +#include "config.hh" + +namespace nix { + +struct ArchiveSettings : Config +{ + Setting<bool> useCaseHack{this, + #if __APPLE__ + true, + #else + false, + #endif + "use-case-hack", + "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; +}; + +static ArchiveSettings archiveSettings; + +static GlobalConfig::Register r1(&archiveSettings); + +const std::string narVersionMagic1 = "nix-archive-1"; + +static string caseHackSuffix = "~nix~case~hack~"; + +PathFilter defaultPathFilter = [](const Path &) { return true; }; + + +static void dumpContents(const Path & path, size_t size, + Sink & sink) +{ + sink << "contents" << size; + + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) throw SysError(format("opening file '%1%'") % path); + + std::vector<unsigned char> buf(65536); + size_t left = size; + + while (left > 0) { + auto n = std::min(left, buf.size()); + readFull(fd.get(), buf.data(), n); + left -= n; + sink(buf.data(), n); + } + + writePadding(size, sink); +} + + +static void dump(const Path & path, Sink & sink, PathFilter & filter) +{ + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); + + sink << "("; + + if (S_ISREG(st.st_mode)) { + sink << "type" << "regular"; + if (st.st_mode & S_IXUSR) + sink << "executable" << ""; + dumpContents(path, (size_t) st.st_size, sink); + } + + else if (S_ISDIR(st.st_mode)) { + sink << "type" << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map<string, string> unhacked; + for (auto & i : readDirectory(path)) + if (archiveSettings.useCaseHack) { + string name(i.name); + size_t pos = i.name.find(caseHackSuffix); + if (pos != string::npos) { + debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name)); + name.erase(pos); + } + if (unhacked.find(name) != unhacked.end()) + throw Error(format("file name collision in between '%1%' and '%2%'") + % (path + "/" + unhacked[name]) % (path + "/" + i.name)); + unhacked[name] = i.name; + } else + unhacked[i.name] = i.name; + + for (auto & i : unhacked) + if (filter(path + "/" + i.first)) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(path + "/" + i.second, sink, filter); + sink << ")"; + } + } + + else if (S_ISLNK(st.st_mode)) + sink << "type" << "symlink" << "target" << readLink(path); + + else throw Error(format("file '%1%' has an unsupported type") % path); + + sink << ")"; +} + + +void dumpPath(const Path & path, Sink & sink, PathFilter & filter) +{ + sink << narVersionMagic1; + dump(path, sink, filter); +} + + +void dumpString(const std::string & s, Sink & sink) +{ + sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")"; +} + + +static SerialisationError badArchive(string s) +{ + return SerialisationError("bad archive: " + s); +} + + +#if 0 +static void skipGeneric(Source & source) +{ + if (readString(source) == "(") { + while (readString(source) != ")") + skipGeneric(source); + } +} +#endif + + +static void parseContents(ParseSink & sink, Source & source, const Path & path) +{ + unsigned long long size = readLongLong(source); + + sink.preallocateContents(size); + + unsigned long long left = size; + std::vector<unsigned char> buf(65536); + + while (left) { + checkInterrupt(); + auto n = buf.size(); + if ((unsigned long long)n > left) n = left; + source(buf.data(), n); + sink.receiveContents(buf.data(), n); + left -= n; + } + + readPadding(size, source); +} + + +struct CaseInsensitiveCompare +{ + bool operator() (const string & a, const string & b) const + { + return strcasecmp(a.c_str(), b.c_str()) < 0; + } +}; + + +static void parse(ParseSink & sink, Source & source, const Path & path) +{ + string s; + + s = readString(source); + if (s != "(") throw badArchive("expected open tag"); + + enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; + + std::map<Path, int, CaseInsensitiveCompare> names; + + while (1) { + checkInterrupt(); + + s = readString(source); + + if (s == ")") { + break; + } + + else if (s == "type") { + if (type != tpUnknown) + throw badArchive("multiple type fields"); + string t = readString(source); + + if (t == "regular") { + type = tpRegular; + sink.createRegularFile(path); + } + + else if (t == "directory") { + sink.createDirectory(path); + type = tpDirectory; + } + + else if (t == "symlink") { + type = tpSymlink; + } + + else throw badArchive("unknown file type " + t); + + } + + else if (s == "contents" && type == tpRegular) { + parseContents(sink, source, path); + } + + else if (s == "executable" && type == tpRegular) { + auto s = readString(source); + if (s != "") throw badArchive("executable marker has non-empty value"); + sink.isExecutable(); + } + + else if (s == "entry" && type == tpDirectory) { + string name, prevName; + + s = readString(source); + if (s != "(") throw badArchive("expected open tag"); + + while (1) { + checkInterrupt(); + + s = readString(source); + + if (s == ")") { + break; + } else if (s == "name") { + name = readString(source); + if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos) + throw Error(format("NAR contains invalid file name '%1%'") % name); + if (name <= prevName) + throw Error("NAR directory is not sorted"); + prevName = name; + if (archiveSettings.useCaseHack) { + auto i = names.find(name); + if (i != names.end()) { + debug(format("case collision between '%1%' and '%2%'") % i->first % name); + name += caseHackSuffix; + name += std::to_string(++i->second); + } else + names[name] = 0; + } + } else if (s == "node") { + if (s.empty()) throw badArchive("entry name missing"); + parse(sink, source, path + "/" + name); + } else + throw badArchive("unknown field " + s); + } + } + + else if (s == "target" && type == tpSymlink) { + string target = readString(source); + sink.createSymlink(path, target); + } + + else + throw badArchive("unknown field " + s); + } +} + + +void parseDump(ParseSink & sink, Source & source) +{ + string version; + try { + version = readString(source, narVersionMagic1.size()); + } catch (SerialisationError & e) { + /* This generally means the integer at the start couldn't be + decoded. Ignore and throw the exception below. */ + } + if (version != narVersionMagic1) + throw badArchive("input doesn't look like a Nix archive"); + parse(sink, source, ""); +} + + +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + void createDirectory(const Path & path) + { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError(format("creating directory '%1%'") % p); + }; + + void createRegularFile(const Path & path) + { + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError(format("creating file '%1%'") % p); + } + + void isExecutable() + { + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } + + void preallocateContents(unsigned long long len) + { +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError(format("preallocating file of %1% bytes") % len); + } +#endif + } + + void receiveContents(unsigned char * data, unsigned int len) + { + writeFull(fd.get(), data, len); + } + + void createSymlink(const Path & path, const string & target) + { + Path p = dstPath + path; + nix::createSymlink(target, p); + } +}; + + +void restorePath(const Path & path, Source & source) +{ + RestoreSink sink; + sink.dstPath = path; + parseDump(sink, source); +} + + +void copyNAR(Source & source, Sink & sink) +{ + // FIXME: if 'source' is the output of dumpPath() followed by EOF, + // we should just forward all data directly without parsing. + + ParseSink parseSink; /* null sink; just parse the NAR */ + + LambdaSource wrapper([&](unsigned char * data, size_t len) { + auto n = source.read(data, len); + sink(data, n); + return n; + }); + + parseDump(parseSink, wrapper); +} + + +} diff --git a/third_party/nix/src/libutil/archive.hh b/third_party/nix/src/libutil/archive.hh new file mode 100644 index 000000000000..25be426c1a4d --- /dev/null +++ b/third_party/nix/src/libutil/archive.hh @@ -0,0 +1,84 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" + + +namespace nix { + + +/* dumpPath creates a Nix archive of the specified path. The format + is as follows: + + IF path points to a REGULAR FILE: + dump(path) = attrs( + [ ("type", "regular") + , ("contents", contents(path)) + ]) + + IF path points to a DIRECTORY: + dump(path) = attrs( + [ ("type", "directory") + , ("entries", concat(map(f, sort(entries(path))))) + ]) + where f(fn) = attrs( + [ ("name", fn) + , ("file", dump(path + "/" + fn)) + ]) + + where: + + attrs(as) = concat(map(attr, as)) + encN(0) + attrs((a, b)) = encS(a) + encS(b) + + encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) + + encN(n) = 64-bit little-endian encoding of n. + + contents(path) = the contents of a regular file. + + sort(strings) = lexicographic sort by 8-bit value (strcmp). + + entries(path) = the entries of a directory, without `.' and + `..'. + + `+' denotes string concatenation. */ + + +void dumpPath(const Path & path, Sink & sink, + PathFilter & filter = defaultPathFilter); + +void dumpString(const std::string & s, Sink & sink); + +/* FIXME: fix this API, it sucks. */ +struct ParseSink +{ + virtual void createDirectory(const Path & path) { }; + + virtual void createRegularFile(const Path & path) { }; + virtual void isExecutable() { }; + virtual void preallocateContents(unsigned long long size) { }; + virtual void receiveContents(unsigned char * data, unsigned int len) { }; + + virtual void createSymlink(const Path & path, const string & target) { }; +}; + +struct TeeSink : ParseSink +{ + TeeSource source; + + TeeSink(Source & source) : source(source) { } +}; + +void parseDump(ParseSink & sink, Source & source); + +void restorePath(const Path & path, Source & source); + +/* Read a NAR from 'source' and write it to 'sink'. */ +void copyNAR(Source & source, Sink & sink); + + +extern const std::string narVersionMagic1; + + +} diff --git a/third_party/nix/src/libutil/args.cc b/third_party/nix/src/libutil/args.cc new file mode 100644 index 000000000000..7af2a1bf731a --- /dev/null +++ b/third_party/nix/src/libutil/args.cc @@ -0,0 +1,203 @@ +#include "args.hh" +#include "hash.hh" + +namespace nix { + +Args::FlagMaker Args::mkFlag() +{ + return FlagMaker(*this); +} + +Args::FlagMaker::~FlagMaker() +{ + assert(flag->longName != ""); + args.longFlags[flag->longName] = flag; + if (flag->shortName) args.shortFlags[flag->shortName] = flag; +} + +void Args::parseCmdline(const Strings & _cmdline) +{ + Strings pendingArgs; + bool dashDash = false; + + Strings cmdline(_cmdline); + + for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { + + auto arg = *pos; + + /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f', + `-j3` -> `-j 3`). */ + if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) { + *pos = (string) "-" + arg[1]; + auto next = pos; ++next; + for (unsigned int j = 2; j < arg.length(); j++) + if (isalpha(arg[j])) + cmdline.insert(next, (string) "-" + arg[j]); + else { + cmdline.insert(next, string(arg, j)); + break; + } + arg = *pos; + } + + if (!dashDash && arg == "--") { + dashDash = true; + ++pos; + } + else if (!dashDash && std::string(arg, 0, 1) == "-") { + if (!processFlag(pos, cmdline.end())) + throw UsageError(format("unrecognised flag '%1%'") % arg); + } + else { + pendingArgs.push_back(*pos++); + if (processArgs(pendingArgs, false)) + pendingArgs.clear(); + } + } + + processArgs(pendingArgs, true); +} + +void Args::printHelp(const string & programName, std::ostream & out) +{ + std::cout << "Usage: " << programName << " <FLAGS>..."; + for (auto & exp : expectedArgs) { + std::cout << renderLabels({exp.label}); + // FIXME: handle arity > 1 + if (exp.arity == 0) std::cout << "..."; + if (exp.optional) std::cout << "?"; + } + std::cout << "\n"; + + auto s = description(); + if (s != "") + std::cout << "\nSummary: " << s << ".\n"; + + if (longFlags.size()) { + std::cout << "\n"; + std::cout << "Flags:\n"; + printFlags(out); + } +} + +void Args::printFlags(std::ostream & out) +{ + Table2 table; + for (auto & flag : longFlags) { + if (hiddenCategories.count(flag.second->category)) continue; + table.push_back(std::make_pair( + (flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : " ") + + "--" + flag.first + renderLabels(flag.second->labels), + flag.second->description)); + } + printTable(out, table); +} + +bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + assert(pos != end); + + auto process = [&](const std::string & name, const Flag & flag) -> bool { + ++pos; + std::vector<std::string> args; + for (size_t n = 0 ; n < flag.arity; ++n) { + if (pos == end) { + if (flag.arity == ArityAny) break; + throw UsageError(format("flag '%1%' requires %2% argument(s)") + % name % flag.arity); + } + args.push_back(*pos++); + } + flag.handler(std::move(args)); + return true; + }; + + if (string(*pos, 0, 2) == "--") { + auto i = longFlags.find(string(*pos, 2)); + if (i == longFlags.end()) return false; + return process("--" + i->first, *i->second); + } + + if (string(*pos, 0, 1) == "-" && pos->size() == 2) { + auto c = (*pos)[1]; + auto i = shortFlags.find(c); + if (i == shortFlags.end()) return false; + return process(std::string("-") + c, *i->second); + } + + return false; +} + +bool Args::processArgs(const Strings & args, bool finish) +{ + if (expectedArgs.empty()) { + if (!args.empty()) + throw UsageError(format("unexpected argument '%1%'") % args.front()); + return true; + } + + auto & exp = expectedArgs.front(); + + bool res = false; + + if ((exp.arity == 0 && finish) || + (exp.arity > 0 && args.size() == exp.arity)) + { + std::vector<std::string> ss; + for (auto & s : args) ss.push_back(s); + exp.handler(std::move(ss)); + expectedArgs.pop_front(); + res = true; + } + + if (finish && !expectedArgs.empty() && !expectedArgs.front().optional) + throw UsageError("more arguments are required"); + + return res; +} + +Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht) +{ + arity(1); + label("type"); + description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')"); + handler([ht](std::string s) { + *ht = parseHashType(s); + if (*ht == htUnknown) + throw UsageError("unknown hash type '%1%'", s); + }); + return *this; +} + +Strings argvToStrings(int argc, char * * argv) +{ + Strings args; + argc--; argv++; + while (argc--) args.push_back(*argv++); + return args; +} + +std::string renderLabels(const Strings & labels) +{ + std::string res; + for (auto label : labels) { + for (auto & c : label) c = std::toupper(c); + res += " <" + label + ">"; + } + return res; +} + +void printTable(std::ostream & out, const Table2 & table) +{ + size_t max = 0; + for (auto & row : table) + max = std::max(max, row.first.size()); + for (auto & row : table) { + out << " " << row.first + << std::string(max - row.first.size() + 2, ' ') + << row.second << "\n"; + } +} + +} diff --git a/third_party/nix/src/libutil/args.hh b/third_party/nix/src/libutil/args.hh new file mode 100644 index 000000000000..ad5fcca39418 --- /dev/null +++ b/third_party/nix/src/libutil/args.hh @@ -0,0 +1,201 @@ +#pragma once + +#include <iostream> +#include <map> +#include <memory> + +#include "util.hh" + +namespace nix { + +MakeError(UsageError, Error); + +enum HashType : char; + +class Args +{ +public: + + /* Parse the command line, throwing a UsageError if something goes + wrong. */ + void parseCmdline(const Strings & cmdline); + + virtual void printHelp(const string & programName, std::ostream & out); + + virtual std::string description() { return ""; } + +protected: + + static const size_t ArityAny = std::numeric_limits<size_t>::max(); + + /* Flags. */ + struct Flag + { + typedef std::shared_ptr<Flag> ptr; + std::string longName; + char shortName = 0; + std::string description; + Strings labels; + size_t arity = 0; + std::function<void(std::vector<std::string>)> handler; + std::string category; + }; + + std::map<std::string, Flag::ptr> longFlags; + std::map<char, Flag::ptr> shortFlags; + + virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); + + virtual void printFlags(std::ostream & out); + + /* Positional arguments. */ + struct ExpectedArg + { + std::string label; + size_t arity; // 0 = any + bool optional; + std::function<void(std::vector<std::string>)> handler; + }; + + std::list<ExpectedArg> expectedArgs; + + virtual bool processArgs(const Strings & args, bool finish); + + std::set<std::string> hiddenCategories; + +public: + + class FlagMaker + { + Args & args; + Flag::ptr flag; + friend class Args; + FlagMaker(Args & args) : args(args), flag(std::make_shared<Flag>()) { }; + public: + ~FlagMaker(); + FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; }; + FlagMaker & shortName(char s) { flag->shortName = s; return *this; }; + FlagMaker & description(const std::string & s) { flag->description = s; return *this; }; + FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; }; + FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; }; + FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; }; + FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; }; + FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; }; + FlagMaker & handler(std::function<void(std::string)> handler) { + flag->arity = 1; + flag->handler = [handler](std::vector<std::string> ss) { handler(std::move(ss[0])); }; + return *this; + }; + FlagMaker & category(const std::string & s) { flag->category = s; return *this; }; + + template<class T> + FlagMaker & dest(T * dest) + { + flag->arity = 1; + flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; }; + return *this; + }; + + template<class T> + FlagMaker & set(T * dest, const T & val) + { + flag->arity = 0; + flag->handler = [=](std::vector<std::string> ss) { *dest = val; }; + return *this; + }; + + FlagMaker & mkHashTypeFlag(HashType * ht); + }; + + FlagMaker mkFlag(); + + /* Helper functions for constructing flags / positional + arguments. */ + + void mkFlag1(char shortName, const std::string & longName, + const std::string & label, const std::string & description, + std::function<void(std::string)> fun) + { + mkFlag() + .shortName(shortName) + .longName(longName) + .labels({label}) + .description(description) + .arity(1) + .handler([=](std::vector<std::string> ss) { fun(ss[0]); }); + } + + void mkFlag(char shortName, const std::string & name, + const std::string & description, bool * dest) + { + mkFlag(shortName, name, description, dest, true); + } + + template<class T> + void mkFlag(char shortName, const std::string & longName, const std::string & description, + T * dest, const T & value) + { + mkFlag() + .shortName(shortName) + .longName(longName) + .description(description) + .handler([=](std::vector<std::string> ss) { *dest = value; }); + } + + template<class I> + void mkIntFlag(char shortName, const std::string & longName, + const std::string & description, I * dest) + { + mkFlag<I>(shortName, longName, description, [=](I n) { + *dest = n; + }); + } + + template<class I> + void mkFlag(char shortName, const std::string & longName, + const std::string & description, std::function<void(I)> fun) + { + mkFlag() + .shortName(shortName) + .longName(longName) + .labels({"N"}) + .description(description) + .arity(1) + .handler([=](std::vector<std::string> ss) { + I n; + if (!string2Int(ss[0], n)) + throw UsageError("flag '--%s' requires a integer argument", longName); + fun(n); + }); + } + + /* Expect a string argument. */ + void expectArg(const std::string & label, string * dest, bool optional = false) + { + expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector<std::string> ss) { + *dest = ss[0]; + }}); + } + + /* Expect 0 or more arguments. */ + void expectArgs(const std::string & label, std::vector<std::string> * dest) + { + expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector<std::string> ss) { + *dest = std::move(ss); + }}); + } + + friend class MultiCommand; +}; + +Strings argvToStrings(int argc, char * * argv); + +/* Helper function for rendering argument labels. */ +std::string renderLabels(const Strings & labels); + +/* Helper function for printing 2-column tables. */ +typedef std::vector<std::pair<std::string, std::string>> Table2; + +void printTable(std::ostream & out, const Table2 & table); + +} diff --git a/third_party/nix/src/libutil/compression.cc b/third_party/nix/src/libutil/compression.cc new file mode 100644 index 000000000000..0dd84e32034a --- /dev/null +++ b/third_party/nix/src/libutil/compression.cc @@ -0,0 +1,432 @@ +#include "compression.hh" +#include "util.hh" +#include "finally.hh" +#include "logging.hh" + +#include <lzma.h> +#include <bzlib.h> +#include <cstdio> +#include <cstring> + +#include <brotli/decode.h> +#include <brotli/encode.h> + +#include <iostream> + +namespace nix { + +// Don't feed brotli too much at once. +struct ChunkedCompressionSink : CompressionSink +{ + uint8_t outbuf[32 * 1024]; + + void write(const unsigned char * data, size_t len) override + { + const size_t CHUNK_SIZE = sizeof(outbuf) << 2; + while (len) { + size_t n = std::min(CHUNK_SIZE, len); + writeInternal(data, n); + data += n; + len -= n; + } + } + + virtual void writeInternal(const unsigned char * data, size_t len) = 0; +}; + +struct NoneSink : CompressionSink +{ + Sink & nextSink; + NoneSink(Sink & nextSink) : nextSink(nextSink) { } + void finish() override { flush(); } + void write(const unsigned char * data, size_t len) override { nextSink(data, len); } +}; + +struct XzDecompressionSink : CompressionSink +{ + Sink & nextSink; + uint8_t outbuf[BUFSIZ]; + lzma_stream strm = LZMA_STREAM_INIT; + bool finished = false; + + XzDecompressionSink(Sink & nextSink) : nextSink(nextSink) + { + lzma_ret ret = lzma_stream_decoder( + &strm, UINT64_MAX, LZMA_CONCATENATED); + if (ret != LZMA_OK) + throw CompressionError("unable to initialise lzma decoder"); + + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + ~XzDecompressionSink() + { + lzma_end(&strm); + } + + void finish() override + { + CompressionSink::flush(); + write(nullptr, 0); + } + + void write(const unsigned char * data, size_t len) override + { + strm.next_in = data; + strm.avail_in = len; + + while (!finished && (!data || strm.avail_in)) { + checkInterrupt(); + + lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + throw CompressionError("error %d while decompressing xz file", ret); + + finished = ret == LZMA_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BzipDecompressionSink : ChunkedCompressionSink +{ + Sink & nextSink; + bz_stream strm; + bool finished = false; + + BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink) + { + memset(&strm, 0, sizeof(strm)); + int ret = BZ2_bzDecompressInit(&strm, 0, 0); + if (ret != BZ_OK) + throw CompressionError("unable to initialise bzip2 decoder"); + + strm.next_out = (char *) outbuf; + strm.avail_out = sizeof(outbuf); + } + + ~BzipDecompressionSink() + { + BZ2_bzDecompressEnd(&strm); + } + + void finish() override + { + flush(); + write(nullptr, 0); + } + + void writeInternal(const unsigned char * data, size_t len) override + { + assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max()); + + strm.next_in = (char *) data; + strm.avail_in = len; + + while (strm.avail_in) { + checkInterrupt(); + + int ret = BZ2_bzDecompress(&strm); + if (ret != BZ_OK && ret != BZ_STREAM_END) + throw CompressionError("error while decompressing bzip2 file"); + + finished = ret == BZ_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = (char *) outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BrotliDecompressionSink : ChunkedCompressionSink +{ + Sink & nextSink; + BrotliDecoderState * state; + bool finished = false; + + BrotliDecompressionSink(Sink & nextSink) : nextSink(nextSink) + { + state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + if (!state) + throw CompressionError("unable to initialize brotli decoder"); + } + + ~BrotliDecompressionSink() + { + BrotliDecoderDestroyInstance(state); + } + + void finish() override + { + flush(); + writeInternal(nullptr, 0); + } + + void writeInternal(const unsigned char * data, size_t len) override + { + const uint8_t * next_in = data; + size_t avail_in = len; + uint8_t * next_out = outbuf; + size_t avail_out = sizeof(outbuf); + + while (!finished && (!data || avail_in)) { + checkInterrupt(); + + if (!BrotliDecoderDecompressStream(state, + &avail_in, &next_in, + &avail_out, &next_out, + nullptr)) + throw CompressionError("error while decompressing brotli file"); + + if (avail_out < sizeof(outbuf) || avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - avail_out); + next_out = outbuf; + avail_out = sizeof(outbuf); + } + + finished = BrotliDecoderIsFinished(state); + } + } +}; + +ref<std::string> decompress(const std::string & method, const std::string & in) +{ + StringSink ssink; + auto sink = makeDecompressionSink(method, ssink); + (*sink)(in); + sink->finish(); + return ssink.s; +} + +ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink) +{ + if (method == "none" || method == "") + return make_ref<NoneSink>(nextSink); + else if (method == "xz") + return make_ref<XzDecompressionSink>(nextSink); + else if (method == "bzip2") + return make_ref<BzipDecompressionSink>(nextSink); + else if (method == "br") + return make_ref<BrotliDecompressionSink>(nextSink); + else + throw UnknownCompressionMethod("unknown compression method '%s'", method); +} + +struct XzCompressionSink : CompressionSink +{ + Sink & nextSink; + uint8_t outbuf[BUFSIZ]; + lzma_stream strm = LZMA_STREAM_INIT; + bool finished = false; + + XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink) + { + lzma_ret ret; + bool done = false; + + if (parallel) { +#ifdef HAVE_LZMA_MT + lzma_mt mt_options = {}; + mt_options.flags = 0; + mt_options.timeout = 300; // Using the same setting as the xz cmd line + mt_options.preset = LZMA_PRESET_DEFAULT; + mt_options.filters = NULL; + mt_options.check = LZMA_CHECK_CRC64; + mt_options.threads = lzma_cputhreads(); + mt_options.block_size = 0; + if (mt_options.threads == 0) + mt_options.threads = 1; + // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the + // number of threads. + ret = lzma_stream_encoder_mt(&strm, &mt_options); + done = true; +#else + printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression"); +#endif + } + + if (!done) + ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); + + if (ret != LZMA_OK) + throw CompressionError("unable to initialise lzma encoder"); + + // FIXME: apply the x86 BCJ filter? + + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + ~XzCompressionSink() + { + lzma_end(&strm); + } + + void finish() override + { + CompressionSink::flush(); + write(nullptr, 0); + } + + void write(const unsigned char * data, size_t len) override + { + strm.next_in = data; + strm.avail_in = len; + + while (!finished && (!data || strm.avail_in)) { + checkInterrupt(); + + lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + throw CompressionError("error %d while compressing xz file", ret); + + finished = ret == LZMA_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BzipCompressionSink : ChunkedCompressionSink +{ + Sink & nextSink; + bz_stream strm; + bool finished = false; + + BzipCompressionSink(Sink & nextSink) : nextSink(nextSink) + { + memset(&strm, 0, sizeof(strm)); + int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); + if (ret != BZ_OK) + throw CompressionError("unable to initialise bzip2 encoder"); + + strm.next_out = (char *) outbuf; + strm.avail_out = sizeof(outbuf); + } + + ~BzipCompressionSink() + { + BZ2_bzCompressEnd(&strm); + } + + void finish() override + { + flush(); + writeInternal(nullptr, 0); + } + + void writeInternal(const unsigned char * data, size_t len) override + { + assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max()); + + strm.next_in = (char *) data; + strm.avail_in = len; + + while (!finished && (!data || strm.avail_in)) { + checkInterrupt(); + + int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH); + if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END) + throw CompressionError("error %d while compressing bzip2 file", ret); + + finished = ret == BZ_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = (char *) outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BrotliCompressionSink : ChunkedCompressionSink +{ + Sink & nextSink; + uint8_t outbuf[BUFSIZ]; + BrotliEncoderState *state; + bool finished = false; + + BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink) + { + state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); + if (!state) + throw CompressionError("unable to initialise brotli encoder"); + } + + ~BrotliCompressionSink() + { + BrotliEncoderDestroyInstance(state); + } + + void finish() override + { + flush(); + writeInternal(nullptr, 0); + } + + void writeInternal(const unsigned char * data, size_t len) override + { + const uint8_t * next_in = data; + size_t avail_in = len; + uint8_t * next_out = outbuf; + size_t avail_out = sizeof(outbuf); + + while (!finished && (!data || avail_in)) { + checkInterrupt(); + + if (!BrotliEncoderCompressStream(state, + data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, + &avail_in, &next_in, + &avail_out, &next_out, + nullptr)) + throw CompressionError("error while compressing brotli compression"); + + if (avail_out < sizeof(outbuf) || avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - avail_out); + next_out = outbuf; + avail_out = sizeof(outbuf); + } + + finished = BrotliEncoderIsFinished(state); + } + } +}; + +ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) +{ + if (method == "none") + return make_ref<NoneSink>(nextSink); + else if (method == "xz") + return make_ref<XzCompressionSink>(nextSink, parallel); + else if (method == "bzip2") + return make_ref<BzipCompressionSink>(nextSink); + else if (method == "br") + return make_ref<BrotliCompressionSink>(nextSink); + else + throw UnknownCompressionMethod(format("unknown compression method '%s'") % method); +} + +ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel) +{ + StringSink ssink; + auto sink = makeCompressionSink(method, ssink, parallel); + (*sink)(in); + sink->finish(); + return ssink.s; +} + +} diff --git a/third_party/nix/src/libutil/compression.hh b/third_party/nix/src/libutil/compression.hh new file mode 100644 index 000000000000..dd666a4e19fd --- /dev/null +++ b/third_party/nix/src/libutil/compression.hh @@ -0,0 +1,28 @@ +#pragma once + +#include "ref.hh" +#include "types.hh" +#include "serialise.hh" + +#include <string> + +namespace nix { + +struct CompressionSink : BufferedSink +{ + virtual void finish() = 0; +}; + +ref<std::string> decompress(const std::string & method, const std::string & in); + +ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink); + +ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false); + +ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false); + +MakeError(UnknownCompressionMethod, Error); + +MakeError(CompressionError, Error); + +} diff --git a/third_party/nix/src/libutil/config.cc b/third_party/nix/src/libutil/config.cc new file mode 100644 index 000000000000..9023cb1bb6de --- /dev/null +++ b/third_party/nix/src/libutil/config.cc @@ -0,0 +1,338 @@ +#include "config.hh" +#include "args.hh" +#include "json.hh" + +namespace nix { + +bool Config::set(const std::string & name, const std::string & value) +{ + auto i = _settings.find(name); + if (i == _settings.end()) return false; + i->second.setting->set(value); + i->second.setting->overriden = true; + return true; +} + +void Config::addSetting(AbstractSetting * setting) +{ + _settings.emplace(setting->name, Config::SettingData(false, setting)); + for (auto & alias : setting->aliases) + _settings.emplace(alias, Config::SettingData(true, setting)); + + bool set = false; + + auto i = unknownSettings.find(setting->name); + if (i != unknownSettings.end()) { + setting->set(i->second); + setting->overriden = true; + unknownSettings.erase(i); + set = true; + } + + for (auto & alias : setting->aliases) { + auto i = unknownSettings.find(alias); + if (i != unknownSettings.end()) { + if (set) + warn("setting '%s' is set, but it's an alias of '%s' which is also set", + alias, setting->name); + else { + setting->set(i->second); + setting->overriden = true; + unknownSettings.erase(i); + set = true; + } + } + } +} + +void AbstractConfig::warnUnknownSettings() +{ + for (auto & s : unknownSettings) + warn("unknown setting '%s'", s.first); +} + +void AbstractConfig::reapplyUnknownSettings() +{ + auto unknownSettings2 = std::move(unknownSettings); + for (auto & s : unknownSettings2) + set(s.first, s.second); +} + +void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly) +{ + for (auto & opt : _settings) + if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden)) + res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); +} + +void AbstractConfig::applyConfigFile(const Path & path) +{ + try { + string contents = readFile(path); + + unsigned int pos = 0; + + while (pos < contents.size()) { + string line; + while (pos < contents.size() && contents[pos] != '\n') + line += contents[pos++]; + pos++; + + string::size_type hash = line.find('#'); + if (hash != string::npos) + line = string(line, 0, hash); + + vector<string> tokens = tokenizeString<vector<string> >(line); + if (tokens.empty()) continue; + + if (tokens.size() < 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + + auto include = false; + auto ignoreMissing = false; + if (tokens[0] == "include") + include = true; + else if (tokens[0] == "!include") { + include = true; + ignoreMissing = true; + } + + if (include) { + if (tokens.size() != 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + auto p = absPath(tokens[1], dirOf(path)); + if (pathExists(p)) { + applyConfigFile(p); + } else if (!ignoreMissing) { + throw Error("file '%1%' included from '%2%' not found", p, path); + } + continue; + } + + if (tokens[1] != "=") + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + + string name = tokens[0]; + + vector<string>::iterator i = tokens.begin(); + advance(i, 2); + + set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + }; + } catch (SysError &) { } +} + +void Config::resetOverriden() +{ + for (auto & s : _settings) + s.second.setting->overriden = false; +} + +void Config::toJSON(JSONObject & out) +{ + for (auto & s : _settings) + if (!s.second.isAlias) { + JSONObject out2(out.object(s.first)); + out2.attr("description", s.second.setting->description); + JSONPlaceholder out3(out2.placeholder("value")); + s.second.setting->toJSON(out3); + } +} + +void Config::convertToArgs(Args & args, const std::string & category) +{ + for (auto & s : _settings) + if (!s.second.isAlias) + s.second.setting->convertToArg(args, category); +} + +AbstractSetting::AbstractSetting( + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases) + : name(name), description(description), aliases(aliases) +{ +} + +void AbstractSetting::toJSON(JSONPlaceholder & out) +{ + out.write(to_string()); +} + +void AbstractSetting::convertToArg(Args & args, const std::string & category) +{ +} + +template<typename T> +void BaseSetting<T>::toJSON(JSONPlaceholder & out) +{ + out.write(value); +} + +template<typename T> +void BaseSetting<T>::convertToArg(Args & args, const std::string & category) +{ + args.mkFlag() + .longName(name) + .description(description) + .arity(1) + .handler([=](std::vector<std::string> ss) { overriden = true; set(ss[0]); }) + .category(category); +} + +template<> void BaseSetting<std::string>::set(const std::string & str) +{ + value = str; +} + +template<> std::string BaseSetting<std::string>::to_string() +{ + return value; +} + +template<typename T> +void BaseSetting<T>::set(const std::string & str) +{ + static_assert(std::is_integral<T>::value, "Integer required."); + if (!string2Int(str, value)) + throw UsageError("setting '%s' has invalid value '%s'", name, str); +} + +template<typename T> +std::string BaseSetting<T>::to_string() +{ + static_assert(std::is_integral<T>::value, "Integer required."); + return std::to_string(value); +} + +template<> void BaseSetting<bool>::set(const std::string & str) +{ + if (str == "true" || str == "yes" || str == "1") + value = true; + else if (str == "false" || str == "no" || str == "0") + value = false; + else + throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); +} + +template<> std::string BaseSetting<bool>::to_string() +{ + return value ? "true" : "false"; +} + +template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string & category) +{ + args.mkFlag() + .longName(name) + .description(description) + .handler([=](std::vector<std::string> ss) { override(true); }) + .category(category); + args.mkFlag() + .longName("no-" + name) + .description(description) + .handler([=](std::vector<std::string> ss) { override(false); }) + .category(category); +} + +template<> void BaseSetting<Strings>::set(const std::string & str) +{ + value = tokenizeString<Strings>(str); +} + +template<> std::string BaseSetting<Strings>::to_string() +{ + return concatStringsSep(" ", value); +} + +template<> void BaseSetting<Strings>::toJSON(JSONPlaceholder & out) +{ + JSONList list(out.list()); + for (auto & s : value) + list.elem(s); +} + +template<> void BaseSetting<StringSet>::set(const std::string & str) +{ + value = tokenizeString<StringSet>(str); +} + +template<> std::string BaseSetting<StringSet>::to_string() +{ + return concatStringsSep(" ", value); +} + +template<> void BaseSetting<StringSet>::toJSON(JSONPlaceholder & out) +{ + JSONList list(out.list()); + for (auto & s : value) + list.elem(s); +} + +template class BaseSetting<int>; +template class BaseSetting<unsigned int>; +template class BaseSetting<long>; +template class BaseSetting<unsigned long>; +template class BaseSetting<long long>; +template class BaseSetting<unsigned long long>; +template class BaseSetting<bool>; +template class BaseSetting<std::string>; +template class BaseSetting<Strings>; +template class BaseSetting<StringSet>; + +void PathSetting::set(const std::string & str) +{ + if (str == "") { + if (allowEmpty) + value = ""; + else + throw UsageError("setting '%s' cannot be empty", name); + } else + value = canonPath(str); +} + +bool GlobalConfig::set(const std::string & name, const std::string & value) +{ + for (auto & config : *configRegistrations) + if (config->set(name, value)) return true; + + unknownSettings.emplace(name, value); + + return false; +} + +void GlobalConfig::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly) +{ + for (auto & config : *configRegistrations) + config->getSettings(res, overridenOnly); +} + +void GlobalConfig::resetOverriden() +{ + for (auto & config : *configRegistrations) + config->resetOverriden(); +} + +void GlobalConfig::toJSON(JSONObject & out) +{ + for (auto & config : *configRegistrations) + config->toJSON(out); +} + +void GlobalConfig::convertToArgs(Args & args, const std::string & category) +{ + for (auto & config : *configRegistrations) + config->convertToArgs(args, category); +} + +GlobalConfig globalConfig; + +GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; + +GlobalConfig::Register::Register(Config * config) +{ + if (!configRegistrations) + configRegistrations = new ConfigRegistrations; + configRegistrations->emplace_back(config); +} + +} diff --git a/third_party/nix/src/libutil/config.hh b/third_party/nix/src/libutil/config.hh new file mode 100644 index 000000000000..d86c65ff033a --- /dev/null +++ b/third_party/nix/src/libutil/config.hh @@ -0,0 +1,261 @@ +#include <map> +#include <set> + +#include "types.hh" + +#pragma once + +namespace nix { + +class Args; +class AbstractSetting; +class JSONPlaceholder; +class JSONObject; + +class AbstractConfig +{ +protected: + StringMap unknownSettings; + + AbstractConfig(const StringMap & initials = {}) + : unknownSettings(initials) + { } + +public: + + virtual bool set(const std::string & name, const std::string & value) = 0; + + struct SettingInfo + { + std::string value; + std::string description; + }; + + virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0; + + void applyConfigFile(const Path & path); + + virtual void resetOverriden() = 0; + + virtual void toJSON(JSONObject & out) = 0; + + virtual void convertToArgs(Args & args, const std::string & category) = 0; + + void warnUnknownSettings(); + + void reapplyUnknownSettings(); +}; + +/* A class to simplify providing configuration settings. The typical + use is to inherit Config and add Setting<T> members: + + class MyClass : private Config + { + Setting<int> foo{this, 123, "foo", "the number of foos to use"}; + Setting<std::string> bar{this, "blabla", "bar", "the name of the bar"}; + + MyClass() : Config(readConfigFile("/etc/my-app.conf")) + { + std::cout << foo << "\n"; // will print 123 unless overriden + } + }; +*/ + +class Config : public AbstractConfig +{ + friend class AbstractSetting; + +public: + + struct SettingData + { + bool isAlias; + AbstractSetting * setting; + SettingData(bool isAlias, AbstractSetting * setting) + : isAlias(isAlias), setting(setting) + { } + }; + + typedef std::map<std::string, SettingData> Settings; + +private: + + Settings _settings; + +public: + + Config(const StringMap & initials = {}) + : AbstractConfig(initials) + { } + + bool set(const std::string & name, const std::string & value) override; + + void addSetting(AbstractSetting * setting); + + void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override; + + void resetOverriden() override; + + void toJSON(JSONObject & out) override; + + void convertToArgs(Args & args, const std::string & category) override; +}; + +class AbstractSetting +{ + friend class Config; + +public: + + const std::string name; + const std::string description; + const std::set<std::string> aliases; + + int created = 123; + + bool overriden = false; + +protected: + + AbstractSetting( + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases); + + virtual ~AbstractSetting() + { + // Check against a gcc miscompilation causing our constructor + // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). + assert(created == 123); + } + + virtual void set(const std::string & value) = 0; + + virtual std::string to_string() = 0; + + virtual void toJSON(JSONPlaceholder & out); + + virtual void convertToArg(Args & args, const std::string & category); + + bool isOverriden() { return overriden; } +}; + +/* A setting of type T. */ +template<typename T> +class BaseSetting : public AbstractSetting +{ +protected: + + T value; + +public: + + BaseSetting(const T & def, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}) + : AbstractSetting(name, description, aliases) + , value(def) + { } + + operator const T &() const { return value; } + operator T &() { return value; } + const T & get() const { return value; } + bool operator ==(const T & v2) const { return value == v2; } + bool operator !=(const T & v2) const { return value != v2; } + void operator =(const T & v) { assign(v); } + virtual void assign(const T & v) { value = v; } + + void set(const std::string & str) override; + + virtual void override(const T & v) + { + overriden = true; + value = v; + } + + std::string to_string() override; + + void convertToArg(Args & args, const std::string & category) override; + + void toJSON(JSONPlaceholder & out) override; +}; + +template<typename T> +std::ostream & operator <<(std::ostream & str, const BaseSetting<T> & opt) +{ + str << (const T &) opt; + return str; +} + +template<typename T> +bool operator ==(const T & v1, const BaseSetting<T> & v2) { return v1 == (const T &) v2; } + +template<typename T> +class Setting : public BaseSetting<T> +{ +public: + Setting(Config * options, + const T & def, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}) + : BaseSetting<T>(def, name, description, aliases) + { + options->addSetting(this); + } + + void operator =(const T & v) { this->assign(v); } +}; + +/* A special setting for Paths. These are automatically canonicalised + (e.g. "/foo//bar/" becomes "/foo/bar"). */ +class PathSetting : public BaseSetting<Path> +{ + bool allowEmpty; + +public: + + PathSetting(Config * options, + bool allowEmpty, + const Path & def, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}) + : BaseSetting<Path>(def, name, description, aliases) + , allowEmpty(allowEmpty) + { + options->addSetting(this); + } + + void set(const std::string & str) override; + + Path operator +(const char * p) const { return value + p; } + + void operator =(const Path & v) { this->assign(v); } +}; + +struct GlobalConfig : public AbstractConfig +{ + typedef std::vector<Config*> ConfigRegistrations; + static ConfigRegistrations * configRegistrations; + + bool set(const std::string & name, const std::string & value) override; + + void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override; + + void resetOverriden() override; + + void toJSON(JSONObject & out) override; + + void convertToArgs(Args & args, const std::string & category) override; + + struct Register + { + Register(Config * config); + }; +}; + +extern GlobalConfig globalConfig; + +} diff --git a/third_party/nix/src/libutil/finally.hh b/third_party/nix/src/libutil/finally.hh new file mode 100644 index 000000000000..7760cfe9a410 --- /dev/null +++ b/third_party/nix/src/libutil/finally.hh @@ -0,0 +1,14 @@ +#pragma once + +#include <functional> + +/* A trivial class to run a function at the end of a scope. */ +class Finally +{ +private: + std::function<void()> fun; + +public: + Finally(std::function<void()> fun) : fun(fun) { } + ~Finally() { fun(); } +}; diff --git a/third_party/nix/src/libutil/hash.cc b/third_party/nix/src/libutil/hash.cc new file mode 100644 index 000000000000..1c14ebb187cc --- /dev/null +++ b/third_party/nix/src/libutil/hash.cc @@ -0,0 +1,355 @@ +#include <iostream> +#include <cstring> + +#include <openssl/md5.h> +#include <openssl/sha.h> + +#include "hash.hh" +#include "archive.hh" +#include "util.hh" +#include "istringstream_nocopy.hh" + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +namespace nix { + + +void Hash::init() +{ + if (type == htMD5) hashSize = md5HashSize; + else if (type == htSHA1) hashSize = sha1HashSize; + else if (type == htSHA256) hashSize = sha256HashSize; + else if (type == htSHA512) hashSize = sha512HashSize; + else abort(); + assert(hashSize <= maxHashSize); + memset(hash, 0, maxHashSize); +} + + +bool Hash::operator == (const Hash & h2) const +{ + if (hashSize != h2.hashSize) return false; + for (unsigned int i = 0; i < hashSize; i++) + if (hash[i] != h2.hash[i]) return false; + return true; +} + + +bool Hash::operator != (const Hash & h2) const +{ + return !(*this == h2); +} + + +bool Hash::operator < (const Hash & h) const +{ + if (hashSize < h.hashSize) return true; + if (hashSize > h.hashSize) return false; + for (unsigned int i = 0; i < hashSize; i++) { + if (hash[i] < h.hash[i]) return true; + if (hash[i] > h.hash[i]) return false; + } + return false; +} + + +const string base16Chars = "0123456789abcdef"; + + +static string printHash16(const Hash & hash) +{ + char buf[hash.hashSize * 2]; + for (unsigned int i = 0; i < hash.hashSize; i++) { + buf[i * 2] = base16Chars[hash.hash[i] >> 4]; + buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f]; + } + return string(buf, hash.hashSize * 2); +} + + +// omitted: E O U T +const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; + + +static string printHash32(const Hash & hash) +{ + assert(hash.hashSize); + size_t len = hash.base32Len(); + assert(len); + + string s; + s.reserve(len); + + for (int n = (int) len - 1; n >= 0; n--) { + unsigned int b = n * 5; + unsigned int i = b / 8; + unsigned int j = b % 8; + unsigned char c = + (hash.hash[i] >> j) + | (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j)); + s.push_back(base32Chars[c & 0x1f]); + } + + return s; +} + + +string printHash16or32(const Hash & hash) +{ + return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); +} + + +std::string Hash::to_string(Base base, bool includeType) const +{ + std::string s; + if (base == SRI || includeType) { + s += printHashType(type); + s += base == SRI ? '-' : ':'; + } + switch (base) { + case Base16: + s += printHash16(*this); + break; + case Base32: + s += printHash32(*this); + break; + case Base64: + case SRI: + s += base64Encode(std::string((const char *) hash, hashSize)); + break; + } + return s; +} + + +Hash::Hash(const std::string & s, HashType type) + : type(type) +{ + size_t pos = 0; + bool isSRI = false; + + auto sep = s.find(':'); + if (sep == string::npos) { + sep = s.find('-'); + if (sep != string::npos) { + isSRI = true; + } else if (type == htUnknown) + throw BadHash("hash '%s' does not include a type", s); + } + + if (sep != string::npos) { + string hts = string(s, 0, sep); + this->type = parseHashType(hts); + if (this->type == htUnknown) + throw BadHash("unknown hash type '%s'", hts); + if (type != htUnknown && type != this->type) + throw BadHash("hash '%s' should have type '%s'", s, printHashType(type)); + pos = sep + 1; + } + + init(); + + size_t size = s.size() - pos; + + if (!isSRI && size == base16Len()) { + + auto parseHexDigit = [&](char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + throw BadHash("invalid base-16 hash '%s'", s); + }; + + for (unsigned int i = 0; i < hashSize; i++) { + hash[i] = + parseHexDigit(s[pos + i * 2]) << 4 + | parseHexDigit(s[pos + i * 2 + 1]); + } + } + + else if (!isSRI && size == base32Len()) { + + for (unsigned int n = 0; n < size; ++n) { + char c = s[pos + size - n - 1]; + unsigned char digit; + for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ + if (base32Chars[digit] == c) break; + if (digit >= 32) + throw BadHash("invalid base-32 hash '%s'", s); + unsigned int b = n * 5; + unsigned int i = b / 8; + unsigned int j = b % 8; + hash[i] |= digit << j; + + if (i < hashSize - 1) { + hash[i + 1] |= digit >> (8 - j); + } else { + if (digit >> (8 - j)) + throw BadHash("invalid base-32 hash '%s'", s); + } + } + } + + else if (isSRI || size == base64Len()) { + auto d = base64Decode(std::string(s, pos)); + if (d.size() != hashSize) + throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s); + assert(hashSize); + memcpy(hash, d.data(), hashSize); + } + + else + throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type)); +} + + +union Ctx +{ + MD5_CTX md5; + SHA_CTX sha1; + SHA256_CTX sha256; + SHA512_CTX sha512; +}; + + +static void start(HashType ht, Ctx & ctx) +{ + if (ht == htMD5) MD5_Init(&ctx.md5); + else if (ht == htSHA1) SHA1_Init(&ctx.sha1); + else if (ht == htSHA256) SHA256_Init(&ctx.sha256); + else if (ht == htSHA512) SHA512_Init(&ctx.sha512); +} + + +static void update(HashType ht, Ctx & ctx, + const unsigned char * bytes, size_t len) +{ + if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); + else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); + else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len); + else if (ht == htSHA512) SHA512_Update(&ctx.sha512, bytes, len); +} + + +static void finish(HashType ht, Ctx & ctx, unsigned char * hash) +{ + if (ht == htMD5) MD5_Final(hash, &ctx.md5); + else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1); + else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256); + else if (ht == htSHA512) SHA512_Final(hash, &ctx.sha512); +} + + +Hash hashString(HashType ht, const string & s) +{ + Ctx ctx; + Hash hash(ht); + start(ht, ctx); + update(ht, ctx, (const unsigned char *) s.data(), s.length()); + finish(ht, ctx, hash.hash); + return hash; +} + + +Hash hashFile(HashType ht, const Path & path) +{ + Ctx ctx; + Hash hash(ht); + start(ht, ctx); + + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) throw SysError(format("opening file '%1%'") % path); + + std::vector<unsigned char> buf(8192); + ssize_t n; + while ((n = read(fd.get(), buf.data(), buf.size()))) { + checkInterrupt(); + if (n == -1) throw SysError(format("reading file '%1%'") % path); + update(ht, ctx, buf.data(), n); + } + + finish(ht, ctx, hash.hash); + return hash; +} + + +HashSink::HashSink(HashType ht) : ht(ht) +{ + ctx = new Ctx; + bytes = 0; + start(ht, *ctx); +} + +HashSink::~HashSink() +{ + bufPos = 0; + delete ctx; +} + +void HashSink::write(const unsigned char * data, size_t len) +{ + bytes += len; + update(ht, *ctx, data, len); +} + +HashResult HashSink::finish() +{ + flush(); + Hash hash(ht); + nix::finish(ht, *ctx, hash.hash); + return HashResult(hash, bytes); +} + +HashResult HashSink::currentHash() +{ + flush(); + Ctx ctx2 = *ctx; + Hash hash(ht); + nix::finish(ht, ctx2, hash.hash); + return HashResult(hash, bytes); +} + + +HashResult hashPath( + HashType ht, const Path & path, PathFilter & filter) +{ + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish(); +} + + +Hash compressHash(const Hash & hash, unsigned int newSize) +{ + Hash h; + h.hashSize = newSize; + for (unsigned int i = 0; i < hash.hashSize; ++i) + h.hash[i % newSize] ^= hash.hash[i]; + return h; +} + + +HashType parseHashType(const string & s) +{ + if (s == "md5") return htMD5; + else if (s == "sha1") return htSHA1; + else if (s == "sha256") return htSHA256; + else if (s == "sha512") return htSHA512; + else return htUnknown; +} + + +string printHashType(HashType ht) +{ + if (ht == htMD5) return "md5"; + else if (ht == htSHA1) return "sha1"; + else if (ht == htSHA256) return "sha256"; + else if (ht == htSHA512) return "sha512"; + else abort(); +} + + +} diff --git a/third_party/nix/src/libutil/hash.hh b/third_party/nix/src/libutil/hash.hh new file mode 100644 index 000000000000..2dbc3b630814 --- /dev/null +++ b/third_party/nix/src/libutil/hash.hh @@ -0,0 +1,131 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" + + +namespace nix { + + +MakeError(BadHash, Error); + + +enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 }; + + +const int md5HashSize = 16; +const int sha1HashSize = 20; +const int sha256HashSize = 32; +const int sha512HashSize = 64; + +extern const string base32Chars; + +enum Base : int { Base64, Base32, Base16, SRI }; + + +struct Hash +{ + static const unsigned int maxHashSize = 64; + unsigned int hashSize = 0; + unsigned char hash[maxHashSize] = {}; + + HashType type = htUnknown; + + /* Create an unset hash object. */ + Hash() { }; + + /* Create a zero-filled hash object. */ + Hash(HashType type) : type(type) { init(); }; + + /* Initialize the hash from a string representation, in the format + "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a + Subresource Integrity hash expression). If the 'type' argument + is htUnknown, then the hash type must be specified in the + string. */ + Hash(const std::string & s, HashType type = htUnknown); + + void init(); + + /* Check whether a hash is set. */ + operator bool () const { return type != htUnknown; } + + /* Check whether two hash are equal. */ + bool operator == (const Hash & h2) const; + + /* Check whether two hash are not equal. */ + bool operator != (const Hash & h2) const; + + /* For sorting. */ + bool operator < (const Hash & h) const; + + /* Returns the length of a base-16 representation of this hash. */ + size_t base16Len() const + { + return hashSize * 2; + } + + /* Returns the length of a base-32 representation of this hash. */ + size_t base32Len() const + { + return (hashSize * 8 - 1) / 5 + 1; + } + + /* Returns the length of a base-64 representation of this hash. */ + size_t base64Len() const + { + return ((4 * hashSize / 3) + 3) & ~3; + } + + /* Return a string representation of the hash, in base-16, base-32 + or base-64. By default, this is prefixed by the hash type + (e.g. "sha256:"). */ + std::string to_string(Base base = Base32, bool includeType = true) const; +}; + + +/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ +string printHash16or32(const Hash & hash); + +/* Compute the hash of the given string. */ +Hash hashString(HashType ht, const string & s); + +/* Compute the hash of the given file. */ +Hash hashFile(HashType ht, const Path & path); + +/* Compute the hash of the given path. The hash is defined as + (essentially) hashString(ht, dumpPath(path)). */ +typedef std::pair<Hash, unsigned long long> HashResult; +HashResult hashPath(HashType ht, const Path & path, + PathFilter & filter = defaultPathFilter); + +/* Compress a hash to the specified number of bytes by cyclically + XORing bytes together. */ +Hash compressHash(const Hash & hash, unsigned int newSize); + +/* Parse a string representing a hash type. */ +HashType parseHashType(const string & s); + +/* And the reverse. */ +string printHashType(HashType ht); + + +union Ctx; + +class HashSink : public BufferedSink +{ +private: + HashType ht; + Ctx * ctx; + unsigned long long bytes; + +public: + HashSink(HashType ht); + HashSink(const HashSink & h); + ~HashSink(); + void write(const unsigned char * data, size_t len); + HashResult finish(); + HashResult currentHash(); +}; + + +} diff --git a/third_party/nix/src/libutil/istringstream_nocopy.hh b/third_party/nix/src/libutil/istringstream_nocopy.hh new file mode 100644 index 000000000000..f7beac578e39 --- /dev/null +++ b/third_party/nix/src/libutil/istringstream_nocopy.hh @@ -0,0 +1,92 @@ +/* This file provides a variant of std::istringstream that doesn't + copy its string argument. This is useful for large strings. The + caller must ensure that the string object is not destroyed while + it's referenced by this object. */ + +#pragma once + +#include <string> +#include <iostream> + +template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>> +class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits> +{ +public: + typedef std::basic_string<CharT, Traits, Allocator> string_type; + + typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type; + + typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type; + + typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type; + + typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type; + +private: + const string_type & s; + + off_type off; + +public: + basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0} + { + } + +private: + pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which) + { + if (which & std::ios_base::in) { + this->off = dir == std::ios_base::beg + ? off + : (dir == std::ios_base::end + ? s.size() + off + : this->off + off); + } + return pos_type(this->off); + } + + pos_type seekpos(pos_type pos, std::ios_base::openmode which) + { + return seekoff(pos, std::ios_base::beg, which); + } + + std::streamsize showmanyc() + { + return s.size() - off; + } + + int_type underflow() + { + if (typename string_type::size_type(off) == s.size()) + return traits_type::eof(); + return traits_type::to_int_type(s[off]); + } + + int_type uflow() + { + if (typename string_type::size_type(off) == s.size()) + return traits_type::eof(); + return traits_type::to_int_type(s[off++]); + } + + int_type pbackfail(int_type ch) + { + if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1])) + return traits_type::eof(); + + return traits_type::to_int_type(s[--off]); + } + +}; + +template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>> +class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits> +{ + typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type; + buf_type buf; +public: + basic_istringstream_nocopy(const typename buf_type::string_type & s) : + std::basic_iostream<CharT, Traits>(&buf), buf(s) {}; +}; + +typedef basic_istringstream_nocopy<char> istringstream_nocopy; diff --git a/third_party/nix/src/libutil/json.cc b/third_party/nix/src/libutil/json.cc new file mode 100644 index 000000000000..0a6fb65f0605 --- /dev/null +++ b/third_party/nix/src/libutil/json.cc @@ -0,0 +1,174 @@ +#include "json.hh" + +#include <iomanip> +#include <cstring> + +namespace nix { + +void toJSON(std::ostream & str, const char * start, const char * end) +{ + str << '"'; + for (auto i = start; i != end; i++) + if (*i == '\"' || *i == '\\') str << '\\' << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else if (*i >= 0 && *i < 32) + str << "\\u" << std::setfill('0') << std::setw(4) << std::hex << (uint16_t) *i << std::dec; + else str << *i; + str << '"'; +} + +void toJSON(std::ostream & str, const char * s) +{ + if (!s) str << "null"; else toJSON(str, s, s + strlen(s)); +} + +template<> void toJSON<int>(std::ostream & str, const int & n) { str << n; } +template<> void toJSON<unsigned int>(std::ostream & str, const unsigned int & n) { str << n; } +template<> void toJSON<long>(std::ostream & str, const long & n) { str << n; } +template<> void toJSON<unsigned long>(std::ostream & str, const unsigned long & n) { str << n; } +template<> void toJSON<long long>(std::ostream & str, const long long & n) { str << n; } +template<> void toJSON<unsigned long long>(std::ostream & str, const unsigned long long & n) { str << n; } +template<> void toJSON<float>(std::ostream & str, const float & n) { str << n; } +template<> void toJSON<double>(std::ostream & str, const double & n) { str << n; } + +template<> void toJSON<std::string>(std::ostream & str, const std::string & s) +{ + toJSON(str, s.c_str(), s.c_str() + s.size()); +} + +template<> void toJSON<bool>(std::ostream & str, const bool & b) +{ + str << (b ? "true" : "false"); +} + +template<> void toJSON<std::nullptr_t>(std::ostream & str, const std::nullptr_t & b) +{ + str << "null"; +} + +JSONWriter::JSONWriter(std::ostream & str, bool indent) + : state(new JSONState(str, indent)) +{ + state->stack++; +} + +JSONWriter::JSONWriter(JSONState * state) + : state(state) +{ + state->stack++; +} + +JSONWriter::~JSONWriter() +{ + if (state) { + assertActive(); + state->stack--; + if (state->stack == 0) delete state; + } +} + +void JSONWriter::comma() +{ + assertActive(); + if (first) { + first = false; + } else { + state->str << ','; + } + if (state->indent) indent(); +} + +void JSONWriter::indent() +{ + state->str << '\n' << std::string(state->depth * 2, ' '); +} + +void JSONList::open() +{ + state->depth++; + state->str << '['; +} + +JSONList::~JSONList() +{ + state->depth--; + if (state->indent && !first) indent(); + state->str << "]"; +} + +JSONList JSONList::list() +{ + comma(); + return JSONList(state); +} + +JSONObject JSONList::object() +{ + comma(); + return JSONObject(state); +} + +JSONPlaceholder JSONList::placeholder() +{ + comma(); + return JSONPlaceholder(state); +} + +void JSONObject::open() +{ + state->depth++; + state->str << '{'; +} + +JSONObject::~JSONObject() +{ + if (state) { + state->depth--; + if (state->indent && !first) indent(); + state->str << "}"; + } +} + +void JSONObject::attr(const std::string & s) +{ + comma(); + toJSON(state->str, s); + state->str << ':'; + if (state->indent) state->str << ' '; +} + +JSONList JSONObject::list(const std::string & name) +{ + attr(name); + return JSONList(state); +} + +JSONObject JSONObject::object(const std::string & name) +{ + attr(name); + return JSONObject(state); +} + +JSONPlaceholder JSONObject::placeholder(const std::string & name) +{ + attr(name); + return JSONPlaceholder(state); +} + +JSONList JSONPlaceholder::list() +{ + assertValid(); + first = false; + return JSONList(state); +} + +JSONObject JSONPlaceholder::object() +{ + assertValid(); + first = false; + return JSONObject(state); +} + +} diff --git a/third_party/nix/src/libutil/json.hh b/third_party/nix/src/libutil/json.hh new file mode 100644 index 000000000000..02a39917fb5c --- /dev/null +++ b/third_party/nix/src/libutil/json.hh @@ -0,0 +1,189 @@ +#pragma once + +#include <iostream> +#include <vector> +#include <cassert> + +namespace nix { + +void toJSON(std::ostream & str, const char * start, const char * end); +void toJSON(std::ostream & str, const char * s); + +template<typename T> +void toJSON(std::ostream & str, const T & n); + +class JSONWriter +{ +protected: + + struct JSONState + { + std::ostream & str; + bool indent; + size_t depth = 0; + size_t stack = 0; + JSONState(std::ostream & str, bool indent) : str(str), indent(indent) { } + ~JSONState() + { + assert(stack == 0); + } + }; + + JSONState * state; + + bool first = true; + + JSONWriter(std::ostream & str, bool indent); + + JSONWriter(JSONState * state); + + ~JSONWriter(); + + void assertActive() + { + assert(state->stack != 0); + } + + void comma(); + + void indent(); +}; + +class JSONObject; +class JSONPlaceholder; + +class JSONList : JSONWriter +{ +private: + + friend class JSONObject; + friend class JSONPlaceholder; + + void open(); + + JSONList(JSONState * state) + : JSONWriter(state) + { + open(); + } + +public: + + JSONList(std::ostream & str, bool indent = false) + : JSONWriter(str, indent) + { + open(); + } + + ~JSONList(); + + template<typename T> + JSONList & elem(const T & v) + { + comma(); + toJSON(state->str, v); + return *this; + } + + JSONList list(); + + JSONObject object(); + + JSONPlaceholder placeholder(); +}; + +class JSONObject : JSONWriter +{ +private: + + friend class JSONList; + friend class JSONPlaceholder; + + void open(); + + JSONObject(JSONState * state) + : JSONWriter(state) + { + open(); + } + + void attr(const std::string & s); + +public: + + JSONObject(std::ostream & str, bool indent = false) + : JSONWriter(str, indent) + { + open(); + } + + JSONObject(const JSONObject & obj) = delete; + + JSONObject(JSONObject && obj) + : JSONWriter(obj.state) + { + obj.state = 0; + } + + ~JSONObject(); + + template<typename T> + JSONObject & attr(const std::string & name, const T & v) + { + attr(name); + toJSON(state->str, v); + return *this; + } + + JSONList list(const std::string & name); + + JSONObject object(const std::string & name); + + JSONPlaceholder placeholder(const std::string & name); +}; + +class JSONPlaceholder : JSONWriter +{ + +private: + + friend class JSONList; + friend class JSONObject; + + JSONPlaceholder(JSONState * state) + : JSONWriter(state) + { + } + + void assertValid() + { + assertActive(); + assert(first); + } + +public: + + JSONPlaceholder(std::ostream & str, bool indent = false) + : JSONWriter(str, indent) + { + } + + ~JSONPlaceholder() + { + assert(!first || std::uncaught_exception()); + } + + template<typename T> + void write(const T & v) + { + assertValid(); + first = false; + toJSON(state->str, v); + } + + JSONList list(); + + JSONObject object(); +}; + +} diff --git a/third_party/nix/src/libutil/lazy.hh b/third_party/nix/src/libutil/lazy.hh new file mode 100644 index 000000000000..d073e486c2eb --- /dev/null +++ b/third_party/nix/src/libutil/lazy.hh @@ -0,0 +1,48 @@ +#include <exception> +#include <functional> +#include <mutex> + +namespace nix { + +/* A helper class for lazily-initialized variables. + + Lazy<T> var([]() { return value; }); + + declares a variable of type T that is initialized to 'value' (in a + thread-safe way) on first use, that is, when var() is first + called. If the initialiser code throws an exception, then all + subsequent calls to var() will rethrow that exception. */ +template<typename T> +class Lazy +{ + + typedef std::function<T()> Init; + + Init init; + + std::once_flag done; + + T value; + + std::exception_ptr ex; + +public: + + Lazy(Init init) : init(init) + { } + + const T & operator () () + { + std::call_once(done, [&]() { + try { + value = init(); + } catch (...) { + ex = std::current_exception(); + } + }); + if (ex) std::rethrow_exception(ex); + return value; + } +}; + +} diff --git a/third_party/nix/src/libutil/local.mk b/third_party/nix/src/libutil/local.mk new file mode 100644 index 000000000000..e41a67d1f9e1 --- /dev/null +++ b/third_party/nix/src/libutil/local.mk @@ -0,0 +1,9 @@ +libraries += libutil + +libutil_NAME = libnixutil + +libutil_DIR := $(d) + +libutil_SOURCES := $(wildcard $(d)/*.cc) + +libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context diff --git a/third_party/nix/src/libutil/logging.cc b/third_party/nix/src/libutil/logging.cc new file mode 100644 index 000000000000..b379306f6ec0 --- /dev/null +++ b/third_party/nix/src/libutil/logging.cc @@ -0,0 +1,242 @@ +#include "logging.hh" +#include "util.hh" + +#include <atomic> +#include <nlohmann/json.hpp> + +namespace nix { + +static thread_local ActivityId curActivity = 0; + +ActivityId getCurActivity() +{ + return curActivity; +} +void setCurActivity(const ActivityId activityId) +{ + curActivity = activityId; +} + +Logger * logger = makeDefaultLogger(); + +void Logger::warn(const std::string & msg) +{ + log(lvlWarn, ANSI_RED "warning:" ANSI_NORMAL " " + msg); +} + +class SimpleLogger : public Logger +{ +public: + + bool systemd, tty; + + SimpleLogger() + { + systemd = getEnv("IN_SYSTEMD") == "1"; + tty = isatty(STDERR_FILENO); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + + std::string prefix; + + if (systemd) { + char c; + switch (lvl) { + case lvlError: c = '3'; break; + case lvlWarn: c = '4'; break; + case lvlInfo: c = '5'; break; + case lvlTalkative: case lvlChatty: c = '6'; break; + default: c = '7'; + } + prefix = std::string("<") + c + ">"; + } + + writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) + override + { + if (lvl <= verbosity && !s.empty()) + log(lvl, s + "..."); + } +}; + +Verbosity verbosity = lvlInfo; + +void warnOnce(bool & haveWarned, const FormatOrString & fs) +{ + if (!haveWarned) { + warn(fs.s); + haveWarned = true; + } +} + +void writeToStderr(const string & s) +{ + try { + writeFull(STDERR_FILENO, s, false); + } catch (SysError & e) { + /* Ignore failing writes to stderr. We need to ignore write + errors to ensure that cleanup code that logs to stderr runs + to completion if the other side of stderr has been closed + unexpectedly. */ + } +} + +Logger * makeDefaultLogger() +{ + return new SimpleLogger(); +} + +std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32}; + +Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, + const std::string & s, const Logger::Fields & fields, ActivityId parent) + : logger(logger), id(nextId++) +{ + logger.startActivity(id, lvl, type, s, fields, parent); +} + +struct JSONLogger : Logger +{ + Logger & prevLogger; + + JSONLogger(Logger & prevLogger) : prevLogger(prevLogger) { } + + void addFields(nlohmann::json & json, const Fields & fields) + { + if (fields.empty()) return; + auto & arr = json["fields"] = nlohmann::json::array(); + for (auto & f : fields) + if (f.type == Logger::Field::tInt) + arr.push_back(f.i); + else if (f.type == Logger::Field::tString) + arr.push_back(f.s); + else + abort(); + } + + void write(const nlohmann::json & json) + { + prevLogger.log(lvlError, "@nix " + json.dump()); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + nlohmann::json json; + json["action"] = "msg"; + json["level"] = lvl; + json["msg"] = fs.s; + write(json); + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) override + { + nlohmann::json json; + json["action"] = "start"; + json["id"] = act; + json["level"] = lvl; + json["type"] = type; + json["text"] = s; + addFields(json, fields); + // FIXME: handle parent + write(json); + } + + void stopActivity(ActivityId act) override + { + nlohmann::json json; + json["action"] = "stop"; + json["id"] = act; + write(json); + } + + void result(ActivityId act, ResultType type, const Fields & fields) override + { + nlohmann::json json; + json["action"] = "result"; + json["id"] = act; + json["type"] = type; + addFields(json, fields); + write(json); + } +}; + +Logger * makeJSONLogger(Logger & prevLogger) +{ + return new JSONLogger(prevLogger); +} + +static Logger::Fields getFields(nlohmann::json & json) +{ + Logger::Fields fields; + for (auto & f : json) { + if (f.type() == nlohmann::json::value_t::number_unsigned) + fields.emplace_back(Logger::Field(f.get<uint64_t>())); + else if (f.type() == nlohmann::json::value_t::string) + fields.emplace_back(Logger::Field(f.get<std::string>())); + else throw Error("unsupported JSON type %d", (int) f.type()); + } + return fields; +} + +bool handleJSONLogMessage(const std::string & msg, + const Activity & act, std::map<ActivityId, Activity> & activities, bool trusted) +{ + if (!hasPrefix(msg, "@nix ")) return false; + + try { + auto json = nlohmann::json::parse(std::string(msg, 5)); + + std::string action = json["action"]; + + if (action == "start") { + auto type = (ActivityType) json["type"]; + if (trusted || type == actDownload) + activities.emplace(std::piecewise_construct, + std::forward_as_tuple(json["id"]), + std::forward_as_tuple(*logger, (Verbosity) json["level"], type, + json["text"], getFields(json["fields"]), act.id)); + } + + else if (action == "stop") + activities.erase((ActivityId) json["id"]); + + else if (action == "result") { + auto i = activities.find((ActivityId) json["id"]); + if (i != activities.end()) + i->second.result((ResultType) json["type"], getFields(json["fields"])); + } + + else if (action == "setPhase") { + std::string phase = json["phase"]; + act.result(resSetPhase, phase); + } + + else if (action == "msg") { + std::string msg = json["msg"]; + logger->log((Verbosity) json["level"], msg); + } + + } catch (std::exception & e) { + printError("bad log message from builder: %s", e.what()); + } + + return true; +} + +Activity::~Activity() { + try { + logger.stopActivity(id); + } catch (...) { + ignoreException(); + } +} + +} diff --git a/third_party/nix/src/libutil/logging.hh b/third_party/nix/src/libutil/logging.hh new file mode 100644 index 000000000000..5df03da74e00 --- /dev/null +++ b/third_party/nix/src/libutil/logging.hh @@ -0,0 +1,172 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +typedef enum { + lvlError = 0, + lvlWarn, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit +} Verbosity; + +typedef enum { + actUnknown = 0, + actCopyPath = 100, + actDownload = 101, + actRealise = 102, + actCopyPaths = 103, + actBuilds = 104, + actBuild = 105, + actOptimiseStore = 106, + actVerifyPaths = 107, + actSubstitute = 108, + actQueryPathInfo = 109, + actPostBuildHook = 110, +} ActivityType; + +typedef enum { + resFileLinked = 100, + resBuildLogLine = 101, + resUntrustedPath = 102, + resCorruptedPath = 103, + resSetPhase = 104, + resProgress = 105, + resSetExpected = 106, + resPostBuildLogLine = 107, +} ResultType; + +typedef uint64_t ActivityId; + +class Logger +{ + friend struct Activity; + +public: + + struct Field + { + // FIXME: use std::variant. + enum { tInt = 0, tString = 1 } type; + uint64_t i = 0; + std::string s; + Field(const std::string & s) : type(tString), s(s) { } + Field(const char * s) : type(tString), s(s) { } + Field(const uint64_t & i) : type(tInt), i(i) { } + }; + + typedef std::vector<Field> Fields; + + virtual ~Logger() { } + + virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; + + void log(const FormatOrString & fs) + { + log(lvlInfo, fs); + } + + virtual void warn(const std::string & msg); + + virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) { }; + + virtual void stopActivity(ActivityId act) { }; + + virtual void result(ActivityId act, ResultType type, const Fields & fields) { }; +}; + +ActivityId getCurActivity(); +void setCurActivity(const ActivityId activityId); + +struct Activity +{ + Logger & logger; + + const ActivityId id; + + Activity(Logger & logger, Verbosity lvl, ActivityType type, const std::string & s = "", + const Logger::Fields & fields = {}, ActivityId parent = getCurActivity()); + + Activity(Logger & logger, ActivityType type, + const Logger::Fields & fields = {}, ActivityId parent = getCurActivity()) + : Activity(logger, lvlError, type, "", fields, parent) { }; + + Activity(const Activity & act) = delete; + + ~Activity(); + + void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const + { result(resProgress, done, expected, running, failed); } + + void setExpected(ActivityType type2, uint64_t expected) const + { result(resSetExpected, type2, expected); } + + template<typename... Args> + void result(ResultType type, const Args & ... args) const + { + Logger::Fields fields; + nop{(fields.emplace_back(Logger::Field(args)), 1)...}; + result(type, fields); + } + + void result(ResultType type, const Logger::Fields & fields) const + { + logger.result(id, type, fields); + } + + friend class Logger; +}; + +struct PushActivity +{ + const ActivityId prevAct; + PushActivity(ActivityId act) : prevAct(getCurActivity()) { setCurActivity(act); } + ~PushActivity() { setCurActivity(prevAct); } +}; + +extern Logger * logger; + +Logger * makeDefaultLogger(); + +Logger * makeJSONLogger(Logger & prevLogger); + +bool handleJSONLogMessage(const std::string & msg, + const Activity & act, std::map<ActivityId, Activity> & activities, + bool trusted); + +extern Verbosity verbosity; /* suppress msgs > this */ + +/* Print a message if the current log level is at least the specified + level. Note that this has to be implemented as a macro to ensure + that the arguments are evaluated lazily. */ +#define printMsg(level, args...) \ + do { \ + if (level <= nix::verbosity) { \ + logger->log(level, fmt(args)); \ + } \ + } while (0) + +#define printError(args...) printMsg(lvlError, args) +#define printInfo(args...) printMsg(lvlInfo, args) +#define printTalkative(args...) printMsg(lvlTalkative, args) +#define debug(args...) printMsg(lvlDebug, args) +#define vomit(args...) printMsg(lvlVomit, args) + +template<typename... Args> +inline void warn(const std::string & fs, Args... args) +{ + boost::format f(fs); + nop{boost::io::detail::feed(f, args)...}; + logger->warn(f.str()); +} + +void warnOnce(bool & haveWarned, const FormatOrString & fs); + +void writeToStderr(const string & s); + +} diff --git a/third_party/nix/src/libutil/lru-cache.hh b/third_party/nix/src/libutil/lru-cache.hh new file mode 100644 index 000000000000..8b83f842c324 --- /dev/null +++ b/third_party/nix/src/libutil/lru-cache.hh @@ -0,0 +1,92 @@ +#pragma once + +#include <map> +#include <list> +#include <optional> + +namespace nix { + +/* A simple least-recently used cache. Not thread-safe. */ +template<typename Key, typename Value> +class LRUCache +{ +private: + + size_t capacity; + + // Stupid wrapper to get around circular dependency between Data + // and LRU. + struct LRUIterator; + + using Data = std::map<Key, std::pair<LRUIterator, Value>>; + using LRU = std::list<typename Data::iterator>; + + struct LRUIterator { typename LRU::iterator it; }; + + Data data; + LRU lru; + +public: + + LRUCache(size_t capacity) : capacity(capacity) { } + + /* Insert or upsert an item in the cache. */ + void upsert(const Key & key, const Value & value) + { + if (capacity == 0) return; + + erase(key); + + if (data.size() >= capacity) { + /* Retire the oldest item. */ + auto oldest = lru.begin(); + data.erase(*oldest); + lru.erase(oldest); + } + + auto res = data.emplace(key, std::make_pair(LRUIterator(), value)); + assert(res.second); + auto & i(res.first); + + auto j = lru.insert(lru.end(), i); + + i->second.first.it = j; + } + + bool erase(const Key & key) + { + auto i = data.find(key); + if (i == data.end()) return false; + lru.erase(i->second.first.it); + data.erase(i); + return true; + } + + /* Look up an item in the cache. If it exists, it becomes the most + recently used item. */ + std::optional<Value> get(const Key & key) + { + auto i = data.find(key); + if (i == data.end()) return {}; + + /* Move this item to the back of the LRU list. */ + lru.erase(i->second.first.it); + auto j = lru.insert(lru.end(), i); + i->second.first.it = j; + + return i->second.second; + } + + size_t size() + { + return data.size(); + } + + void clear() + { + data.clear(); + lru.clear(); + } +}; + +} diff --git a/third_party/nix/src/libutil/monitor-fd.hh b/third_party/nix/src/libutil/monitor-fd.hh new file mode 100644 index 000000000000..5ee0b88ef50f --- /dev/null +++ b/third_party/nix/src/libutil/monitor-fd.hh @@ -0,0 +1,58 @@ +#pragma once + +#include <thread> +#include <atomic> + +#include <cstdlib> +#include <poll.h> +#include <sys/types.h> +#include <unistd.h> +#include <signal.h> + +namespace nix { + + +class MonitorFdHup +{ +private: + std::thread thread; + +public: + MonitorFdHup(int fd) + { + thread = std::thread([fd]() { + while (true) { + /* Wait indefinitely until a POLLHUP occurs. */ + struct pollfd fds[1]; + fds[0].fd = fd; + /* This shouldn't be necessary, but macOS doesn't seem to + like a zeroed out events field. + See rdar://37537852. + */ + fds[0].events = POLLHUP; + auto count = poll(fds, 1, -1); + if (count == -1) abort(); // can't happen + /* This shouldn't happen, but can on macOS due to a bug. + See rdar://37550628. + + This may eventually need a delay or further + coordination with the main thread if spinning proves + too harmful. + */ + if (count == 0) continue; + assert(fds[0].revents & POLLHUP); + triggerInterrupt(); + break; + } + }); + }; + + ~MonitorFdHup() + { + pthread_cancel(thread.native_handle()); + thread.join(); + } +}; + + +} diff --git a/third_party/nix/src/libutil/pool.hh b/third_party/nix/src/libutil/pool.hh new file mode 100644 index 000000000000..d49067bb95dc --- /dev/null +++ b/third_party/nix/src/libutil/pool.hh @@ -0,0 +1,187 @@ +#pragma once + +#include <functional> +#include <limits> +#include <list> +#include <memory> +#include <cassert> + +#include "sync.hh" +#include "ref.hh" + +namespace nix { + +/* This template class implements a simple pool manager of resources + of some type R, such as database connections. It is used as + follows: + + class Connection { ... }; + + Pool<Connection> pool; + + { + auto conn(pool.get()); + conn->exec("select ..."); + } + + Here, the Connection object referenced by โconnโ is automatically + returned to the pool when โconnโ goes out of scope. +*/ + +template <class R> +class Pool +{ +public: + + /* A function that produces new instances of R on demand. */ + typedef std::function<ref<R>()> Factory; + + /* A function that checks whether an instance of R is still + usable. Unusable instances are removed from the pool. */ + typedef std::function<bool(const ref<R> &)> Validator; + +private: + + Factory factory; + Validator validator; + + struct State + { + size_t inUse = 0; + size_t max; + std::vector<ref<R>> idle; + }; + + Sync<State> state; + + std::condition_variable wakeup; + +public: + + Pool(size_t max = std::numeric_limits<size_t>::max(), + const Factory & factory = []() { return make_ref<R>(); }, + const Validator & validator = [](ref<R> r) { return true; }) + : factory(factory) + , validator(validator) + { + auto state_(state.lock()); + state_->max = max; + } + + void incCapacity() + { + auto state_(state.lock()); + state_->max++; + /* we could wakeup here, but this is only used when we're + * about to nest Pool usages, and we want to save the slot for + * the nested use if we can + */ + } + + void decCapacity() + { + auto state_(state.lock()); + state_->max--; + } + + ~Pool() + { + auto state_(state.lock()); + assert(!state_->inUse); + state_->max = 0; + state_->idle.clear(); + } + + class Handle + { + private: + Pool & pool; + std::shared_ptr<R> r; + bool bad = false; + + friend Pool; + + Handle(Pool & pool, std::shared_ptr<R> r) : pool(pool), r(r) { } + + public: + Handle(Handle && h) : pool(h.pool), r(h.r) { h.r.reset(); } + + Handle(const Handle & l) = delete; + + ~Handle() + { + if (!r) return; + { + auto state_(pool.state.lock()); + if (!bad) + state_->idle.push_back(ref<R>(r)); + assert(state_->inUse); + state_->inUse--; + } + pool.wakeup.notify_one(); + } + + R * operator -> () { return &*r; } + R & operator * () { return *r; } + + void markBad() { bad = true; } + }; + + Handle get() + { + { + auto state_(state.lock()); + + /* If we're over the maximum number of instance, we need + to wait until a slot becomes available. */ + while (state_->idle.empty() && state_->inUse >= state_->max) + state_.wait(wakeup); + + while (!state_->idle.empty()) { + auto p = state_->idle.back(); + state_->idle.pop_back(); + if (validator(p)) { + state_->inUse++; + return Handle(*this, p); + } + } + + state_->inUse++; + } + + /* We need to create a new instance. Because that might take a + while, we don't hold the lock in the meantime. */ + try { + Handle h(*this, factory()); + return h; + } catch (...) { + auto state_(state.lock()); + state_->inUse--; + wakeup.notify_one(); + throw; + } + } + + size_t count() + { + auto state_(state.lock()); + return state_->idle.size() + state_->inUse; + } + + size_t capacity() + { + return state.lock()->max; + } + + void flushBad() + { + auto state_(state.lock()); + std::vector<ref<R>> left; + for (auto & p : state_->idle) + if (validator(p)) + left.push_back(p); + std::swap(state_->idle, left); + } +}; + +} diff --git a/third_party/nix/src/libutil/ref.hh b/third_party/nix/src/libutil/ref.hh new file mode 100644 index 000000000000..0be2a7e74ae0 --- /dev/null +++ b/third_party/nix/src/libutil/ref.hh @@ -0,0 +1,92 @@ +#pragma once + +#include <memory> +#include <exception> +#include <stdexcept> + +namespace nix { + +/* A simple non-nullable reference-counted pointer. Actually a wrapper + around std::shared_ptr that prevents non-null constructions. */ +template<typename T> +class ref +{ +private: + + std::shared_ptr<T> p; + +public: + + ref<T>(const ref<T> & r) + : p(r.p) + { } + + explicit ref<T>(const std::shared_ptr<T> & p) + : p(p) + { + if (!p) + throw std::invalid_argument("null pointer cast to ref"); + } + + explicit ref<T>(T * p) + : p(p) + { + if (!p) + throw std::invalid_argument("null pointer cast to ref"); + } + + T* operator ->() const + { + return &*p; + } + + T& operator *() const + { + return *p; + } + + operator std::shared_ptr<T> () const + { + return p; + } + + std::shared_ptr<T> get_ptr() const + { + return p; + } + + template<typename T2> + ref<T2> cast() const + { + return ref<T2>(std::dynamic_pointer_cast<T2>(p)); + } + + template<typename T2> + std::shared_ptr<T2> dynamic_pointer_cast() const + { + return std::dynamic_pointer_cast<T2>(p); + } + + template<typename T2> + operator ref<T2> () const + { + return ref<T2>((std::shared_ptr<T2>) p); + } + +private: + + template<typename T2, typename... Args> + friend ref<T2> + make_ref(Args&&... args); + +}; + +template<typename T, typename... Args> +inline ref<T> +make_ref(Args&&... args) +{ + auto p = std::make_shared<T>(std::forward<Args>(args)...); + return ref<T>(p); +} + +} diff --git a/third_party/nix/src/libutil/serialise.cc b/third_party/nix/src/libutil/serialise.cc new file mode 100644 index 000000000000..8201549fd7d0 --- /dev/null +++ b/third_party/nix/src/libutil/serialise.cc @@ -0,0 +1,323 @@ +#include "serialise.hh" +#include "util.hh" + +#include <cstring> +#include <cerrno> +#include <memory> + +#include <boost/coroutine2/coroutine.hpp> + + +namespace nix { + + +void BufferedSink::operator () (const unsigned char * data, size_t len) +{ + if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); + + while (len) { + /* Optimisation: bypass the buffer if the data exceeds the + buffer size. */ + if (bufPos + len >= bufSize) { + flush(); + write(data, len); + break; + } + /* Otherwise, copy the bytes to the buffer. Flush the buffer + when it's full. */ + size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; + memcpy(buffer.get() + bufPos, data, n); + data += n; bufPos += n; len -= n; + if (bufPos == bufSize) flush(); + } +} + + +void BufferedSink::flush() +{ + if (bufPos == 0) return; + size_t n = bufPos; + bufPos = 0; // don't trigger the assert() in ~BufferedSink() + write(buffer.get(), n); +} + + +FdSink::~FdSink() +{ + try { flush(); } catch (...) { ignoreException(); } +} + + +size_t threshold = 256 * 1024 * 1024; + +static void warnLargeDump() +{ + printError("warning: dumping very large path (> 256 MiB); this may run out of memory"); +} + + +void FdSink::write(const unsigned char * data, size_t len) +{ + written += len; + static bool warned = false; + if (warn && !warned) { + if (written > threshold) { + warnLargeDump(); + warned = true; + } + } + try { + writeFull(fd, data, len); + } catch (SysError & e) { + _good = false; + throw; + } +} + + +bool FdSink::good() +{ + return _good; +} + + +void Source::operator () (unsigned char * data, size_t len) +{ + while (len) { + size_t n = read(data, len); + data += n; len -= n; + } +} + + +std::string Source::drain() +{ + std::string s; + std::vector<unsigned char> buf(8192); + while (true) { + size_t n; + try { + n = read(buf.data(), buf.size()); + s.append((char *) buf.data(), n); + } catch (EndOfFile &) { + break; + } + } + return s; +} + + +size_t BufferedSource::read(unsigned char * data, size_t len) +{ + if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); + + if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize); + + /* Copy out the data in the buffer. */ + size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; + memcpy(data, buffer.get() + bufPosOut, n); + bufPosOut += n; + if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; + return n; +} + + +bool BufferedSource::hasData() +{ + return bufPosOut < bufPosIn; +} + + +size_t FdSource::readUnbuffered(unsigned char * data, size_t len) +{ + ssize_t n; + do { + checkInterrupt(); + n = ::read(fd, (char *) data, len); + } while (n == -1 && errno == EINTR); + if (n == -1) { _good = false; throw SysError("reading from file"); } + if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } + read += n; + return n; +} + + +bool FdSource::good() +{ + return _good; +} + + +size_t StringSource::read(unsigned char * data, size_t len) +{ + if (pos == s.size()) throw EndOfFile("end of string reached"); + size_t n = s.copy((char *) data, len, pos); + pos += n; + return n; +} + + +#if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600 +#error Coroutines are broken in this version of Boost! +#endif + +std::unique_ptr<Source> sinkToSource( + std::function<void(Sink &)> fun, + std::function<void()> eof) +{ + struct SinkToSource : Source + { + typedef boost::coroutines2::coroutine<std::string> coro_t; + + std::function<void(Sink &)> fun; + std::function<void()> eof; + std::optional<coro_t::pull_type> coro; + bool started = false; + + SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof) + : fun(fun), eof(eof) + { + } + + std::string cur; + size_t pos = 0; + + size_t read(unsigned char * data, size_t len) override + { + if (!coro) + coro = coro_t::pull_type([&](coro_t::push_type & yield) { + LambdaSink sink([&](const unsigned char * data, size_t len) { + if (len) yield(std::string((const char *) data, len)); + }); + fun(sink); + }); + + if (!*coro) { eof(); abort(); } + + if (pos == cur.size()) { + if (!cur.empty()) (*coro)(); + cur = coro->get(); + pos = 0; + } + + auto n = std::min(cur.size() - pos, len); + memcpy(data, (unsigned char *) cur.data() + pos, n); + pos += n; + + return n; + } + }; + + return std::make_unique<SinkToSource>(fun, eof); +} + + +void writePadding(size_t len, Sink & sink) +{ + if (len % 8) { + unsigned char zero[8]; + memset(zero, 0, sizeof(zero)); + sink(zero, 8 - (len % 8)); + } +} + + +void writeString(const unsigned char * buf, size_t len, Sink & sink) +{ + sink << len; + sink(buf, len); + writePadding(len, sink); +} + + +Sink & operator << (Sink & sink, const string & s) +{ + writeString((const unsigned char *) s.data(), s.size(), sink); + return sink; +} + + +template<class T> void writeStrings(const T & ss, Sink & sink) +{ + sink << ss.size(); + for (auto & i : ss) + sink << i; +} + +Sink & operator << (Sink & sink, const Strings & s) +{ + writeStrings(s, sink); + return sink; +} + +Sink & operator << (Sink & sink, const StringSet & s) +{ + writeStrings(s, sink); + return sink; +} + + +void readPadding(size_t len, Source & source) +{ + if (len % 8) { + unsigned char zero[8]; + size_t n = 8 - (len % 8); + source(zero, n); + for (unsigned int i = 0; i < n; i++) + if (zero[i]) throw SerialisationError("non-zero padding"); + } +} + + +size_t readString(unsigned char * buf, size_t max, Source & source) +{ + auto len = readNum<size_t>(source); + if (len > max) throw SerialisationError("string is too long"); + source(buf, len); + readPadding(len, source); + return len; +} + + +string readString(Source & source, size_t max) +{ + auto len = readNum<size_t>(source); + if (len > max) throw SerialisationError("string is too long"); + std::string res(len, 0); + source((unsigned char*) res.data(), len); + readPadding(len, source); + return res; +} + +Source & operator >> (Source & in, string & s) +{ + s = readString(in); + return in; +} + + +template<class T> T readStrings(Source & source) +{ + auto count = readNum<size_t>(source); + T ss; + while (count--) + ss.insert(ss.end(), readString(source)); + return ss; +} + +template Paths readStrings(Source & source); +template PathSet readStrings(Source & source); + + +void StringSink::operator () (const unsigned char * data, size_t len) +{ + static bool warned = false; + if (!warned && s->size() > threshold) { + warnLargeDump(); + warned = true; + } + s->append((const char *) data, len); +} + + +} diff --git a/third_party/nix/src/libutil/serialise.hh b/third_party/nix/src/libutil/serialise.hh new file mode 100644 index 000000000000..a344a5ac7520 --- /dev/null +++ b/third_party/nix/src/libutil/serialise.hh @@ -0,0 +1,337 @@ +#pragma once + +#include <memory> + +#include "types.hh" +#include "util.hh" + + +namespace nix { + + +/* Abstract destination of binary data. */ +struct Sink +{ + virtual ~Sink() { } + virtual void operator () (const unsigned char * data, size_t len) = 0; + virtual bool good() { return true; } + + void operator () (const std::string & s) + { + (*this)((const unsigned char *) s.data(), s.size()); + } +}; + + +/* A buffered abstract sink. */ +struct BufferedSink : Sink +{ + size_t bufSize, bufPos; + std::unique_ptr<unsigned char[]> buffer; + + BufferedSink(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPos(0), buffer(nullptr) { } + + void operator () (const unsigned char * data, size_t len) override; + + void operator () (const std::string & s) + { + Sink::operator()(s); + } + + void flush(); + + virtual void write(const unsigned char * data, size_t len) = 0; +}; + + +/* Abstract source of binary data. */ +struct Source +{ + virtual ~Source() { } + + /* Store exactly โlenโ bytes in the buffer pointed to by โdataโ. + It blocks until all the requested data is available, or throws + an error if it is not going to be available. */ + void operator () (unsigned char * data, size_t len); + + /* Store up to โlenโ in the buffer pointed to by โdataโ, and + return the number of bytes stored. It blocks until at least + one byte is available. */ + virtual size_t read(unsigned char * data, size_t len) = 0; + + virtual bool good() { return true; } + + std::string drain(); +}; + + +/* A buffered abstract source. */ +struct BufferedSource : Source +{ + size_t bufSize, bufPosIn, bufPosOut; + std::unique_ptr<unsigned char[]> buffer; + + BufferedSource(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(nullptr) { } + + size_t read(unsigned char * data, size_t len) override; + + + bool hasData(); + +protected: + /* Underlying read call, to be overridden. */ + virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; +}; + + +/* A sink that writes data to a file descriptor. */ +struct FdSink : BufferedSink +{ + int fd; + bool warn = false; + size_t written = 0; + + FdSink() : fd(-1) { } + FdSink(int fd) : fd(fd) { } + FdSink(FdSink&&) = default; + + FdSink& operator=(FdSink && s) + { + flush(); + fd = s.fd; + s.fd = -1; + warn = s.warn; + written = s.written; + return *this; + } + + ~FdSink(); + + void write(const unsigned char * data, size_t len) override; + + bool good() override; + +private: + bool _good = true; +}; + + +/* A source that reads data from a file descriptor. */ +struct FdSource : BufferedSource +{ + int fd; + size_t read = 0; + + FdSource() : fd(-1) { } + FdSource(int fd) : fd(fd) { } + FdSource(FdSource&&) = default; + + FdSource& operator=(FdSource && s) + { + fd = s.fd; + s.fd = -1; + read = s.read; + return *this; + } + + bool good() override; +protected: + size_t readUnbuffered(unsigned char * data, size_t len) override; +private: + bool _good = true; +}; + + +/* A sink that writes data to a string. */ +struct StringSink : Sink +{ + ref<std::string> s; + StringSink() : s(make_ref<std::string>()) { }; + StringSink(ref<std::string> s) : s(s) { }; + void operator () (const unsigned char * data, size_t len) override; +}; + + +/* A source that reads data from a string. */ +struct StringSource : Source +{ + const string & s; + size_t pos; + StringSource(const string & _s) : s(_s), pos(0) { } + size_t read(unsigned char * data, size_t len) override; +}; + + +/* Adapter class of a Source that saves all data read to `s'. */ +struct TeeSource : Source +{ + Source & orig; + ref<std::string> data; + TeeSource(Source & orig) + : orig(orig), data(make_ref<std::string>()) { } + size_t read(unsigned char * data, size_t len) + { + size_t n = orig.read(data, len); + this->data->append((const char *) data, n); + return n; + } +}; + +/* A reader that consumes the original Source until 'size'. */ +struct SizedSource : Source +{ + Source & orig; + size_t remain; + SizedSource(Source & orig, size_t size) + : orig(orig), remain(size) { } + size_t read(unsigned char * data, size_t len) + { + if (this->remain <= 0) { + throw EndOfFile("sized: unexpected end-of-file"); + } + len = std::min(len, this->remain); + size_t n = this->orig.read(data, len); + this->remain -= n; + return n; + } + + /* Consume the original source until no remain data is left to consume. */ + size_t drainAll() + { + std::vector<unsigned char> buf(8192); + size_t sum = 0; + while (this->remain > 0) { + size_t n = read(buf.data(), buf.size()); + sum += n; + } + return sum; + } +}; + +/* Convert a function into a sink. */ +struct LambdaSink : Sink +{ + typedef std::function<void(const unsigned char *, size_t)> lambda_t; + + lambda_t lambda; + + LambdaSink(const lambda_t & lambda) : lambda(lambda) { } + + virtual void operator () (const unsigned char * data, size_t len) + { + lambda(data, len); + } +}; + + +/* Convert a function into a source. */ +struct LambdaSource : Source +{ + typedef std::function<size_t(unsigned char *, size_t)> lambda_t; + + lambda_t lambda; + + LambdaSource(const lambda_t & lambda) : lambda(lambda) { } + + size_t read(unsigned char * data, size_t len) override + { + return lambda(data, len); + } +}; + + +/* Convert a function that feeds data into a Sink into a Source. The + Source executes the function as a coroutine. */ +std::unique_ptr<Source> sinkToSource( + std::function<void(Sink &)> fun, + std::function<void()> eof = []() { + throw EndOfFile("coroutine has finished"); + }); + + +void writePadding(size_t len, Sink & sink); +void writeString(const unsigned char * buf, size_t len, Sink & sink); + +inline Sink & operator << (Sink & sink, uint64_t n) +{ + unsigned char buf[8]; + buf[0] = n & 0xff; + buf[1] = (n >> 8) & 0xff; + buf[2] = (n >> 16) & 0xff; + buf[3] = (n >> 24) & 0xff; + buf[4] = (n >> 32) & 0xff; + buf[5] = (n >> 40) & 0xff; + buf[6] = (n >> 48) & 0xff; + buf[7] = (unsigned char) (n >> 56) & 0xff; + sink(buf, sizeof(buf)); + return sink; +} + +Sink & operator << (Sink & sink, const string & s); +Sink & operator << (Sink & sink, const Strings & s); +Sink & operator << (Sink & sink, const StringSet & s); + + +MakeError(SerialisationError, Error) + + +template<typename T> +T readNum(Source & source) +{ + unsigned char buf[8]; + source(buf, sizeof(buf)); + + uint64_t n = + ((unsigned long long) buf[0]) | + ((unsigned long long) buf[1] << 8) | + ((unsigned long long) buf[2] << 16) | + ((unsigned long long) buf[3] << 24) | + ((unsigned long long) buf[4] << 32) | + ((unsigned long long) buf[5] << 40) | + ((unsigned long long) buf[6] << 48) | + ((unsigned long long) buf[7] << 56); + + if (n > std::numeric_limits<T>::max()) + throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); + + return (T) n; +} + + +inline unsigned int readInt(Source & source) +{ + return readNum<unsigned int>(source); +} + + +inline uint64_t readLongLong(Source & source) +{ + return readNum<uint64_t>(source); +} + + +void readPadding(size_t len, Source & source); +size_t readString(unsigned char * buf, size_t max, Source & source); +string readString(Source & source, size_t max = std::numeric_limits<size_t>::max()); +template<class T> T readStrings(Source & source); + +Source & operator >> (Source & in, string & s); + +template<typename T> +Source & operator >> (Source & in, T & n) +{ + n = readNum<T>(in); + return in; +} + +template<typename T> +Source & operator >> (Source & in, bool & b) +{ + b = readNum<uint64_t>(in); + return in; +} + + +} diff --git a/third_party/nix/src/libutil/sync.hh b/third_party/nix/src/libutil/sync.hh new file mode 100644 index 000000000000..e1d591d77a84 --- /dev/null +++ b/third_party/nix/src/libutil/sync.hh @@ -0,0 +1,88 @@ +#pragma once + +#include <cstdlib> +#include <mutex> +#include <condition_variable> +#include <cassert> + +namespace nix { + +/* This template class ensures synchronized access to a value of type + T. It is used as follows: + + struct Data { int x; ... }; + + Sync<Data> data; + + { + auto data_(data.lock()); + data_->x = 123; + } + + Here, "data" is automatically unlocked when "data_" goes out of + scope. +*/ + +template<class T, class M = std::mutex> +class Sync +{ +private: + M mutex; + T data; + +public: + + Sync() { } + Sync(const T & data) : data(data) { } + Sync(T && data) noexcept : data(std::move(data)) { } + + class Lock + { + private: + Sync * s; + std::unique_lock<M> lk; + friend Sync; + Lock(Sync * s) : s(s), lk(s->mutex) { } + public: + Lock(Lock && l) : s(l.s) { abort(); } + Lock(const Lock & l) = delete; + ~Lock() { } + T * operator -> () { return &s->data; } + T & operator * () { return s->data; } + + void wait(std::condition_variable & cv) + { + assert(s); + cv.wait(lk); + } + + template<class Rep, class Period> + std::cv_status wait_for(std::condition_variable & cv, + const std::chrono::duration<Rep, Period> & duration) + { + assert(s); + return cv.wait_for(lk, duration); + } + + template<class Rep, class Period, class Predicate> + bool wait_for(std::condition_variable & cv, + const std::chrono::duration<Rep, Period> & duration, + Predicate pred) + { + assert(s); + return cv.wait_for(lk, duration, pred); + } + + template<class Clock, class Duration> + std::cv_status wait_until(std::condition_variable & cv, + const std::chrono::time_point<Clock, Duration> & duration) + { + assert(s); + return cv.wait_until(lk, duration); + } + }; + + Lock lock() { return Lock(this); } +}; + +} diff --git a/third_party/nix/src/libutil/thread-pool.cc b/third_party/nix/src/libutil/thread-pool.cc new file mode 100644 index 000000000000..857ee91f87d0 --- /dev/null +++ b/third_party/nix/src/libutil/thread-pool.cc @@ -0,0 +1,156 @@ +#include "thread-pool.hh" +#include "affinity.hh" + +namespace nix { + +ThreadPool::ThreadPool(size_t _maxThreads) + : maxThreads(_maxThreads) +{ + restoreAffinity(); // FIXME + + if (!maxThreads) { + maxThreads = std::thread::hardware_concurrency(); + if (!maxThreads) maxThreads = 1; + } + + debug("starting pool of %d threads", maxThreads - 1); +} + +ThreadPool::~ThreadPool() +{ + shutdown(); +} + +void ThreadPool::shutdown() +{ + std::vector<std::thread> workers; + { + auto state(state_.lock()); + quit = true; + std::swap(workers, state->workers); + } + + if (workers.empty()) return; + + debug("reaping %d worker threads", workers.size()); + + work.notify_all(); + + for (auto & thr : workers) + thr.join(); +} + +void ThreadPool::enqueue(const work_t & t) +{ + auto state(state_.lock()); + if (quit) + throw ThreadPoolShutDown("cannot enqueue a work item while the thread pool is shutting down"); + state->pending.push(t); + /* Note: process() also executes items, so count it as a worker. */ + if (state->pending.size() > state->workers.size() + 1 && state->workers.size() + 1 < maxThreads) + state->workers.emplace_back(&ThreadPool::doWork, this, false); + work.notify_one(); +} + +void ThreadPool::process() +{ + state_.lock()->draining = true; + + /* Do work until no more work is pending or active. */ + try { + doWork(true); + + auto state(state_.lock()); + + assert(quit); + + if (state->exception) + std::rethrow_exception(state->exception); + + } catch (...) { + /* In the exceptional case, some workers may still be + active. They may be referencing the stack frame of the + caller. So wait for them to finish. (~ThreadPool also does + this, but it might be destroyed after objects referenced by + the work item lambdas.) */ + shutdown(); + throw; + } +} + +void ThreadPool::doWork(bool mainThread) +{ + if (!mainThread) + interruptCheck = [&]() { return (bool) quit; }; + + bool didWork = false; + std::exception_ptr exc; + + while (true) { + work_t w; + { + auto state(state_.lock()); + + if (didWork) { + assert(state->active); + state->active--; + + if (exc) { + + if (!state->exception) { + state->exception = exc; + // Tell the other workers to quit. + quit = true; + work.notify_all(); + } else { + /* Print the exception, since we can't + propagate it. */ + try { + std::rethrow_exception(exc); + } catch (std::exception & e) { + if (!dynamic_cast<Interrupted*>(&e) && + !dynamic_cast<ThreadPoolShutDown*>(&e)) + ignoreException(); + } catch (...) { + } + } + } + } + + /* Wait until a work item is available or we're asked to + quit. */ + while (true) { + if (quit) return; + + if (!state->pending.empty()) break; + + /* If there are no active or pending items, and the + main thread is running process(), then no new items + can be added. So exit. */ + if (!state->active && state->draining) { + quit = true; + work.notify_all(); + return; + } + + state.wait(work); + } + + w = std::move(state->pending.front()); + state->pending.pop(); + state->active++; + } + + try { + w(); + } catch (...) { + exc = std::current_exception(); + } + + didWork = true; + } +} + +} + + diff --git a/third_party/nix/src/libutil/thread-pool.hh b/third_party/nix/src/libutil/thread-pool.hh new file mode 100644 index 000000000000..bb16b639a591 --- /dev/null +++ b/third_party/nix/src/libutil/thread-pool.hh @@ -0,0 +1,143 @@ +#pragma once + +#include "sync.hh" +#include "util.hh" + +#include <queue> +#include <functional> +#include <thread> +#include <map> +#include <atomic> + +namespace nix { + +MakeError(ThreadPoolShutDown, Error) + +/* A simple thread pool that executes a queue of work items + (lambdas). */ +class ThreadPool +{ +public: + + ThreadPool(size_t maxThreads = 0); + + ~ThreadPool(); + + // FIXME: use std::packaged_task? + typedef std::function<void()> work_t; + + /* Enqueue a function to be executed by the thread pool. */ + void enqueue(const work_t & t); + + /* Execute work items until the queue is empty. Note that work + items are allowed to add new items to the queue; this is + handled correctly. Queue processing stops prematurely if any + work item throws an exception. This exception is propagated to + the calling thread. If multiple work items throw an exception + concurrently, only one item is propagated; the others are + printed on stderr and otherwise ignored. */ + void process(); + +private: + + size_t maxThreads; + + struct State + { + std::queue<work_t> pending; + size_t active = 0; + std::exception_ptr exception; + std::vector<std::thread> workers; + bool draining = false; + }; + + std::atomic_bool quit{false}; + + Sync<State> state_; + + std::condition_variable work; + + void doWork(bool mainThread); + + void shutdown(); +}; + +/* Process in parallel a set of items of type T that have a partial + ordering between them. Thus, any item is only processed after all + its dependencies have been processed. */ +template<typename T> +void processGraph( + ThreadPool & pool, + const std::set<T> & nodes, + std::function<std::set<T>(const T &)> getEdges, + std::function<void(const T &)> processNode) +{ + struct Graph { + std::set<T> left; + std::map<T, std::set<T>> refs, rrefs; + }; + + Sync<Graph> graph_(Graph{nodes, {}, {}}); + + std::function<void(const T &)> worker; + + worker = [&](const T & node) { + + { + auto graph(graph_.lock()); + auto i = graph->refs.find(node); + if (i == graph->refs.end()) + goto getRefs; + goto doWork; + } + + getRefs: + { + auto refs = getEdges(node); + refs.erase(node); + + { + auto graph(graph_.lock()); + for (auto & ref : refs) + if (graph->left.count(ref)) { + graph->refs[node].insert(ref); + graph->rrefs[ref].insert(node); + } + if (graph->refs[node].empty()) + goto doWork; + } + } + + return; + + doWork: + processNode(node); + + /* Enqueue work for all nodes that were waiting on this one + and have no unprocessed dependencies. */ + { + auto graph(graph_.lock()); + for (auto & rref : graph->rrefs[node]) { + auto & refs(graph->refs[rref]); + auto i = refs.find(node); + assert(i != refs.end()); + refs.erase(i); + if (refs.empty()) + pool.enqueue(std::bind(worker, rref)); + } + graph->left.erase(node); + graph->refs.erase(node); + graph->rrefs.erase(node); + } + }; + + for (auto & node : nodes) + pool.enqueue(std::bind(worker, std::ref(node))); + + pool.process(); + + if (!graph_.lock()->left.empty()) + throw Error("graph processing incomplete (cyclic reference?)"); +} + +} diff --git a/third_party/nix/src/libutil/types.hh b/third_party/nix/src/libutil/types.hh new file mode 100644 index 000000000000..92bf469b5c6f --- /dev/null +++ b/third_party/nix/src/libutil/types.hh @@ -0,0 +1,150 @@ +#pragma once + + +#include "ref.hh" + +#include <string> +#include <list> +#include <set> +#include <memory> +#include <map> + +#include <boost/format.hpp> + +/* Before 4.7, gcc's std::exception uses empty throw() specifiers for + * its (virtual) destructor and what() in c++11 mode, in violation of spec + */ +#ifdef __GNUC__ +#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) +#define EXCEPTION_NEEDS_THROW_SPEC +#endif +#endif + + +namespace nix { + + +/* Inherit some names from other namespaces for convenience. */ +using std::string; +using std::list; +using std::set; +using std::vector; +using boost::format; + + +/* A variadic template that does nothing. Useful to call a function + for all variadic arguments but ignoring the result. */ +struct nop { template<typename... T> nop(T...) {} }; + + +struct FormatOrString +{ + string s; + FormatOrString(const string & s) : s(s) { }; + FormatOrString(const format & f) : s(f.str()) { }; + FormatOrString(const char * s) : s(s) { }; +}; + + +/* A helper for formatting strings. โfmt(format, a_0, ..., a_n)โ is + equivalent to โboost::format(format) % a_0 % ... % + ... a_nโ. However, โfmt(s)โ is equivalent to โsโ (so no %-expansion + takes place). */ + +inline std::string fmt(const std::string & s) +{ + return s; +} + +inline std::string fmt(const char * s) +{ + return s; +} + +inline std::string fmt(const FormatOrString & fs) +{ + return fs.s; +} + +template<typename... Args> +inline std::string fmt(const std::string & fs, Args... args) +{ + boost::format f(fs); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + nop{boost::io::detail::feed(f, args)...}; + return f.str(); +} + + +/* BaseError should generally not be caught, as it has Interrupted as + a subclass. Catch Error instead. */ +class BaseError : public std::exception +{ +protected: + string prefix_; // used for location traces etc. + string err; +public: + unsigned int status = 1; // exit status + + template<typename... Args> + BaseError(unsigned int status, Args... args) + : err(fmt(args...)) + , status(status) + { + } + + template<typename... Args> + BaseError(Args... args) + : err(fmt(args...)) + { + } + +#ifdef EXCEPTION_NEEDS_THROW_SPEC + ~BaseError() throw () { }; + const char * what() const throw () { return err.c_str(); } +#else + const char * what() const noexcept { return err.c_str(); } +#endif + + const string & msg() const { return err; } + const string & prefix() const { return prefix_; } + BaseError & addPrefix(const FormatOrString & fs); +}; + +#define MakeError(newClass, superClass) \ + class newClass : public superClass \ + { \ + public: \ + using superClass::superClass; \ + }; + +MakeError(Error, BaseError) + +class SysError : public Error +{ +public: + int errNo; + + template<typename... Args> + SysError(Args... args) + : Error(addErrno(fmt(args...))) + { } + +private: + + std::string addErrno(const std::string & s); +}; + + +typedef list<string> Strings; +typedef set<string> StringSet; +typedef std::map<std::string, std::string> StringMap; + + +/* Paths are just strings. */ +typedef string Path; +typedef list<Path> Paths; +typedef set<Path> PathSet; + + +} diff --git a/third_party/nix/src/libutil/util.cc b/third_party/nix/src/libutil/util.cc new file mode 100644 index 000000000000..ad8cc1894c14 --- /dev/null +++ b/third_party/nix/src/libutil/util.cc @@ -0,0 +1,1552 @@ +#include "lazy.hh" +#include "util.hh" +#include "affinity.hh" +#include "sync.hh" +#include "finally.hh" +#include "serialise.hh" + +#include <cctype> +#include <cerrno> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <iostream> +#include <sstream> +#include <thread> +#include <future> + +#include <fcntl.h> +#include <grp.h> +#include <limits.h> +#include <pwd.h> +#include <sys/ioctl.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> + +#ifdef __APPLE__ +#include <sys/syscall.h> +#endif + +#ifdef __linux__ +#include <sys/prctl.h> +#endif + + +extern char * * environ; + + +namespace nix { + + +const std::string nativeSystem = SYSTEM; + + +BaseError & BaseError::addPrefix(const FormatOrString & fs) +{ + prefix_ = fs.s + prefix_; + return *this; +} + + +std::string SysError::addErrno(const std::string & s) +{ + errNo = errno; + return s + ": " + strerror(errNo); +} + + +string getEnv(const string & key, const string & def) +{ + char * value = getenv(key.c_str()); + return value ? string(value) : def; +} + + +std::map<std::string, std::string> getEnv() +{ + std::map<std::string, std::string> env; + for (size_t i = 0; environ[i]; ++i) { + auto s = environ[i]; + auto eq = strchr(s, '='); + if (!eq) + // invalid env, just keep going + continue; + env.emplace(std::string(s, eq), std::string(eq + 1)); + } + return env; +} + + +void clearEnv() +{ + for (auto & name : getEnv()) + unsetenv(name.first.c_str()); +} + +void replaceEnv(std::map<std::string, std::string> newEnv) +{ + clearEnv(); + for (auto newEnvVar : newEnv) + { + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); + } +} + + +Path absPath(Path path, Path dir) +{ + if (path[0] != '/') { + if (dir == "") { +#ifdef __GNU__ + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char *buf = getcwd(NULL, 0); + if (buf == NULL) +#else + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) +#endif + throw SysError("cannot get cwd"); + dir = buf; +#ifdef __GNU__ + free(buf); +#endif + } + path = dir + "/" + path; + } + return canonPath(path); +} + + +Path canonPath(const Path & path, bool resolveSymlinks) +{ + assert(path != ""); + + string s; + + if (path[0] != '/') + throw Error(format("not an absolute path: '%1%'") % path); + + string::const_iterator i = path.begin(), end = path.end(); + string temp; + + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0, maxFollow = 1024; + + while (1) { + + /* Skip slashes. */ + while (i != end && *i == '/') i++; + if (i == end) break; + + /* Ignore `.'. */ + if (*i == '.' && (i + 1 == end || i[1] == '/')) + i++; + + /* If `..', delete the last component. */ + else if (*i == '.' && i + 1 < end && i[1] == '.' && + (i + 2 == end || i[2] == '/')) + { + if (!s.empty()) s.erase(s.rfind('/')); + i += 2; + } + + /* Normal component; copy it. */ + else { + s += '/'; + while (i != end && *i != '/') s += *i++; + + /* If s points to a symlink, resolve it and restart (since + the symlink target might contain new symlinks). */ + if (resolveSymlinks && isLink(s)) { + if (++followCount >= maxFollow) + throw Error(format("infinite symlink recursion in path '%1%'") % path); + temp = absPath(readLink(s), dirOf(s)) + + string(i, end); + i = temp.begin(); /* restart */ + end = temp.end(); + s = ""; + } + } + } + + return s.empty() ? "/" : s; +} + + +Path dirOf(const Path & path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == string::npos) + return "."; + return pos == 0 ? "/" : Path(path, 0, pos); +} + + +string baseNameOf(const Path & path) +{ + if (path.empty()) + return ""; + + Path::size_type last = path.length() - 1; + if (path[last] == '/' && last > 0) + last -= 1; + + Path::size_type pos = path.rfind('/', last); + if (pos == string::npos) + pos = 0; + else + pos += 1; + + return string(path, pos, last - pos + 1); +} + + +bool isInDir(const Path & path, const Path & dir) +{ + return path[0] == '/' + && string(path, 0, dir.size()) == dir + && path.size() >= dir.size() + 2 + && path[dir.size()] == '/'; +} + + +bool isDirOrInDir(const Path & path, const Path & dir) +{ + return path == dir || isInDir(path, dir); +} + + +struct stat lstat(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of '%1%'") % path); + return st; +} + + +bool pathExists(const Path & path) +{ + int res; + struct stat st; + res = lstat(path.c_str(), &st); + if (!res) return true; + if (errno != ENOENT && errno != ENOTDIR) + throw SysError(format("getting status of %1%") % path); + return false; +} + + +Path readLink(const Path & path) +{ + checkInterrupt(); + std::vector<char> buf; + for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { + buf.resize(bufSize); + ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); + if (rlSize == -1) + if (errno == EINVAL) + throw Error("'%1%' is not a symlink", path); + else + throw SysError("reading symbolic link '%1%'", path); + else if (rlSize < bufSize) + return string(buf.data(), rlSize); + } +} + + +bool isLink(const Path & path) +{ + struct stat st = lstat(path); + return S_ISLNK(st.st_mode); +} + + +DirEntries readDirectory(const Path & path) +{ + DirEntries entries; + entries.reserve(64); + + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % path); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + entries.emplace_back(name, dirent->d_ino, +#ifdef HAVE_STRUCT_DIRENT_D_TYPE + dirent->d_type +#else + DT_UNKNOWN +#endif + ); + } + if (errno) throw SysError(format("reading directory '%1%'") % path); + + return entries; +} + + +unsigned char getFileType(const Path & path) +{ + struct stat st = lstat(path); + if (S_ISDIR(st.st_mode)) return DT_DIR; + if (S_ISLNK(st.st_mode)) return DT_LNK; + if (S_ISREG(st.st_mode)) return DT_REG; + return DT_UNKNOWN; +} + + +string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + std::vector<unsigned char> buf(st.st_size); + readFull(fd, buf.data(), st.st_size); + + return string((char *) buf.data(), st.st_size); +} + + +string readFile(const Path & path, bool drain) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError(format("opening file '%1%'") % path); + return drain ? drainFD(fd.get()) : readFile(fd.get()); +} + + +void readFile(const Path & path, Sink & sink) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); +} + + +void writeFile(const Path & path, const string & s, mode_t mode) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError(format("opening file '%1%'") % path); + writeFull(fd.get(), s); +} + + +void writeFile(const Path & path, Source & source, mode_t mode) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError(format("opening file '%1%'") % path); + + std::vector<unsigned char> buf(64 * 1024); + + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), (unsigned char *) buf.data(), n); + } catch (EndOfFile &) { break; } + } +} + + +string readLine(int fd) +{ + string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void writeLine(int fd, string s) +{ + s += '\n'; + writeFull(fd, s); +} + + +static void _deletePath(const Path & path, unsigned long long & bytesFreed) +{ + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + if (errno == ENOENT) return; + throw SysError(format("getting status of '%1%'") % path); + } + + if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) + bytesFreed += st.st_size; + + if (S_ISDIR(st.st_mode)) { + /* Make the directory accessible. */ + const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; + if ((st.st_mode & PERM_MASK) != PERM_MASK) { + if (chmod(path.c_str(), st.st_mode | PERM_MASK) == -1) + throw SysError(format("chmod '%1%'") % path); + } + + for (auto & i : readDirectory(path)) + _deletePath(path + "/" + i.name, bytesFreed); + } + + if (remove(path.c_str()) == -1) { + if (errno == ENOENT) return; + throw SysError(format("cannot unlink '%1%'") % path); + } +} + + +void deletePath(const Path & path) +{ + unsigned long long dummy; + deletePath(path, dummy); +} + + +void deletePath(const Path & path, unsigned long long & bytesFreed) +{ + //Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path); + bytesFreed = 0; + _deletePath(path, bytesFreed); +} + + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + int & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR", "/tmp") : tmpRoot, true); + if (includePid) + return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); + else + return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); +} + + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static int globalCounter = 0; + int localCounter = 0; + int & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError(format("setting group of directory '%1%'") % tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError(format("creating directory '%1%'") % tmpDir); + } +} + + +std::string getUserName() +{ + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER", ""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + + +static Lazy<Path> getHome2([]() { + Path homeDir = getEnv("HOME"); + if (homeDir.empty()) { + std::vector<char> buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + homeDir = pw->pw_dir; + } + return homeDir; +}); + +Path getHome() { return getHome2(); } + + +Path getCacheDir() +{ + Path cacheDir = getEnv("XDG_CACHE_HOME"); + if (cacheDir.empty()) + cacheDir = getHome() + "/.cache"; + return cacheDir; +} + + +Path getConfigDir() +{ + Path configDir = getEnv("XDG_CONFIG_HOME"); + if (configDir.empty()) + configDir = getHome() + "/.config"; + return configDir; +} + +std::vector<Path> getConfigDirs() +{ + Path configHome = getConfigDir(); + string configDirs = getEnv("XDG_CONFIG_DIRS"); + std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + + +Path getDataDir() +{ + Path dataDir = getEnv("XDG_DATA_HOME"); + if (dataDir.empty()) + dataDir = getHome() + "/.local/share"; + return dataDir; +} + + +Paths createDirs(const Path & path) +{ + Paths created; + if (path == "/") return created; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + created = createDirs(dirOf(path)); + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) + throw SysError(format("creating directory '%1%'") % path); + st = lstat(path); + created.push_back(path); + } + + if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) + throw SysError(format("statting symlink '%1%'") % path); + + if (!S_ISDIR(st.st_mode)) throw Error(format("'%1%' is not a directory") % path); + + return created; +} + + +void createSymlink(const Path & target, const Path & link) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target); +} + + +void replaceSymlink(const Path & target, const Path & link) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + if (rename(tmp.c_str(), link.c_str()) != 0) + throw SysError(format("renaming '%1%' to '%2%'") % tmp % link); + + break; + } +} + + +void readFull(int fd, unsigned char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, (char *) buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts) +{ + while (count) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, (char *) buf, count); + if (res == -1 && errno != EINTR) + throw SysError("writing to file"); + if (res > 0) { + count -= res; + buf += res; + } + } +} + + +void writeFull(int fd, const string & s, bool allowInterrupts) +{ + writeFull(fd, (const unsigned char *) s.data(), s.size(), allowInterrupts); +} + + +string drainFD(int fd, bool block) +{ + StringSink sink; + drainFD(fd, sink, block); + return std::move(*sink.s); +} + + +void drainFD(int fd, Sink & sink, bool block) +{ + int saved; + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + std::vector<unsigned char> buf(64 * 1024); + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buf.data(), buf.size()); + if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) + break; + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else sink(buf.data(), rd); + } +} + + + +////////////////////////////////////////////////////////////////////// + + +AutoDelete::AutoDelete() : del{false} {} + +AutoDelete::AutoDelete(const string & p, bool recursive) : path(p) +{ + del = true; + this->recursive = recursive; +} + +AutoDelete::~AutoDelete() +{ + try { + if (del) { + if (recursive) + deletePath(path); + else { + if (remove(path.c_str()) == -1) + throw SysError(format("cannot unlink '%1%'") % path); + } + } + } catch (...) { + ignoreException(); + } +} + +void AutoDelete::cancel() +{ + del = false; +} + +void AutoDelete::reset(const Path & p, bool recursive) { + path = p; + this->recursive = recursive; + del = true; +} + + + +////////////////////////////////////////////////////////////////////// + + +AutoCloseFD::AutoCloseFD() : fd{-1} {} + + +AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} + + +AutoCloseFD::AutoCloseFD(AutoCloseFD&& that) : fd{that.fd} +{ + that.fd = -1; +} + + +AutoCloseFD& AutoCloseFD::operator =(AutoCloseFD&& that) +{ + close(); + fd = that.fd; + that.fd = -1; + return *this; +} + + +AutoCloseFD::~AutoCloseFD() +{ + try { + close(); + } catch (...) { + ignoreException(); + } +} + + +int AutoCloseFD::get() const +{ + return fd; +} + + +void AutoCloseFD::close() +{ + if (fd != -1) { + if (::close(fd) == -1) + /* This should never happen. */ + throw SysError(format("closing file descriptor %1%") % fd); + } +} + + +AutoCloseFD::operator bool() const +{ + return fd != -1; +} + + +int AutoCloseFD::release() +{ + int oldFD = fd; + fd = -1; + return oldFD; +} + + +void Pipe::create() +{ + int fds[2]; +#if HAVE_PIPE2 + if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); +#else + if (pipe(fds) != 0) throw SysError("creating pipe"); + closeOnExec(fds[0]); + closeOnExec(fds[1]); +#endif + readSide = fds[0]; + writeSide = fds[1]; +} + + + +////////////////////////////////////////////////////////////////////// + + +Pid::Pid() +{ +} + + +Pid::Pid(pid_t pid) + : pid(pid) +{ +} + + +Pid::~Pid() +{ + if (pid != -1) kill(); +} + + +void Pid::operator =(pid_t pid) +{ + if (this->pid != -1 && this->pid != pid) kill(); + this->pid = pid; + killSignal = SIGKILL; // reset signal to default +} + + +Pid::operator pid_t() +{ + return pid; +} + + +int Pid::kill() +{ + assert(pid != -1); + + debug(format("killing process %1%") % pid); + + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) { + /* On BSDs, killing a process group will return EPERM if all + processes in the group are zombies (or something like + that). So try to detect and ignore that situation. */ +#if __FreeBSD__ || __APPLE__ + if (errno != EPERM || ::kill(pid, 0) != 0) +#endif + printError((SysError("killing process %d", pid).msg())); + } + + return wait(); +} + + +int Pid::wait() +{ + assert(pid != -1); + while (1) { + int status; + int res = waitpid(pid, &status, 0); + if (res == pid) { + pid = -1; + return status; + } + if (errno != EINTR) + throw SysError("cannot get child exit status"); + checkInterrupt(); + } +} + + +void Pid::setSeparatePG(bool separatePG) +{ + this->separatePG = separatePG; +} + + +void Pid::setKillSignal(int signal) +{ + this->killSignal = signal; +} + + +pid_t Pid::release() +{ + pid_t p = pid; + pid = -1; + return p; +} + + +void killUser(uid_t uid) +{ + debug(format("killing all processes running under uid '%1%'") % uid); + + assert(uid != 0); /* just to be safe... */ + + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ + + ProcessOptions options; + options.allowVfork = false; + + Pid pid = startProcess([&]() { + + if (setuid(uid) == -1) + throw SysError("setting uid"); + + while (true) { +#ifdef __APPLE__ + /* OSX's kill syscall takes a third parameter that, among + other things, determines if kill(-1, signo) affects the + calling process. In the OSX libc, it's set to true, + which means "follow POSIX", which we don't want here + */ + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; +#else + if (kill(-1, SIGKILL) == 0) break; +#endif + if (errno == ESRCH) break; /* no more processes */ + if (errno != EINTR) + throw SysError(format("cannot kill processes for uid '%1%'") % uid); + } + + _exit(0); + }, options); + + int status = pid.wait(); + if (status != 0) + throw Error(format("cannot kill processes for uid '%1%': %2%") % uid % statusToString(status)); + + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ +} + + +////////////////////////////////////////////////////////////////////// + + +/* Wrapper around vfork to prevent the child process from clobbering + the caller's stack frame in the parent. */ +static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline)); +static pid_t doFork(bool allowVfork, std::function<void()> fun) +{ +#ifdef __linux__ + pid_t pid = allowVfork ? vfork() : fork(); +#else + pid_t pid = fork(); +#endif + if (pid != 0) return pid; + fun(); + abort(); +} + + +pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) +{ + auto wrapper = [&]() { + if (!options.allowVfork) + logger = makeDefaultLogger(); + try { +#if __linux__ + if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) + throw SysError("setting death signal"); +#endif + restoreAffinity(); + fun(); + } catch (std::exception & e) { + try { + std::cerr << options.errorPrefix << e.what() << "\n"; + } catch (...) { } + } catch (...) { } + if (options.runExitHandlers) + exit(1); + else + _exit(1); + }; + + pid_t pid = doFork(options.allowVfork, wrapper); + if (pid == -1) throw SysError("unable to fork"); + + return pid; +} + + +std::vector<char *> stringsToCharPtrs(const Strings & ss) +{ + std::vector<char *> res; + for (auto & s : ss) res.push_back((char *) s.c_str()); + res.push_back(0); + return res; +} + + +string runProgram(Path program, bool searchPath, const Strings & args, + const std::optional<std::string> & input) +{ + RunOptions opts(program, args); + opts.searchPath = searchPath; + opts.input = input; + + auto res = runProgram(opts); + + if (!statusOk(res.first)) + throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first))); + + return res.second; +} + +std::pair<int, std::string> runProgram(const RunOptions & options_) +{ + RunOptions options(options_); + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(*sink.s)}; +} + +void runProgram2(const RunOptions & options) +{ + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr<Source> source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique<StringSource>(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + if (options.standardOut) out.create(); + if (source) in.create(); + + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + if (options.environment) + processOptions.allowVfork = false; + + /* Fork. */ + Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); + if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); + if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + + Strings args_(options.args); + args_.push_front(options.program); + + restoreSignals(); + + if (options.searchPath) + execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + else + execv(options.program.c_str(), stringsToCharPtrs(args_).data()); + + throw SysError("executing '%1%'", options.program); + }, processOptions); + + out.writeSide = -1; + + std::thread writerThread; + + std::promise<void> promise; + + Finally doJoin([&]() { + if (writerThread.joinable()) + writerThread.join(); + }); + + + if (source) { + in.readSide = -1; + writerThread = std::thread([&]() { + try { + std::vector<unsigned char> buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), buf.data(), n); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide = -1; + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) promise.get_future().get(); + + if (status) + throw ExecError(status, fmt("program '%1%' %2%", options.program, statusToString(status))); +} + + +void closeMostFDs(const set<int> & exceptions) +{ +#if __linux__ + try { + for (auto & s : readDirectory("/proc/self/fd")) { + auto fd = std::stoi(s.name); + if (!exceptions.count(fd)) { + debug("closing leaked FD %d", fd); + close(fd); + } + } + return; + } catch (SysError &) { + } +#endif + + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (!exceptions.count(fd)) + close(fd); /* ignore result */ +} + + +void closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + + +////////////////////////////////////////////////////////////////////// + + +bool _isInterrupted = false; + +static thread_local bool interruptThrown = false; +thread_local std::function<bool()> interruptCheck; + +void setInterruptThrown() +{ + interruptThrown = true; +} + +void _interrupted() +{ + /* Block user interrupts while an exception is being handled. + Throwing an exception while another exception is being handled + kills the program! */ + if (!interruptThrown && !std::uncaught_exception()) { + interruptThrown = true; + throw Interrupted("interrupted by the user"); + } +} + + +////////////////////////////////////////////////////////////////////// + + +template<class C> C tokenizeString(const string & s, const string & separators) +{ + C result; + string::size_type pos = s.find_first_not_of(separators, 0); + while (pos != string::npos) { + string::size_type end = s.find_first_of(separators, pos + 1); + if (end == string::npos) end = s.size(); + string token(s, pos, end - pos); + result.insert(result.end(), token); + pos = s.find_first_not_of(separators, end); + } + return result; +} + +template Strings tokenizeString(const string & s, const string & separators); +template StringSet tokenizeString(const string & s, const string & separators); +template vector<string> tokenizeString(const string & s, const string & separators); + + +string concatStringsSep(const string & sep, const Strings & ss) +{ + string s; + for (auto & i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + + +string concatStringsSep(const string & sep, const StringSet & ss) +{ + string s; + for (auto & i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + + +string chomp(const string & s) +{ + size_t i = s.find_last_not_of(" \n\r\t"); + return i == string::npos ? "" : string(s, 0, i + 1); +} + + +string trim(const string & s, const string & whitespace) +{ + auto i = s.find_first_not_of(whitespace); + if (i == string::npos) return ""; + auto j = s.find_last_not_of(whitespace); + return string(s, i, j == string::npos ? j : j - i + 1); +} + + +string replaceStrings(const std::string & s, + const std::string & from, const std::string & to) +{ + if (from.empty()) return s; + string res = s; + size_t pos = 0; + while ((pos = res.find(from, pos)) != std::string::npos) { + res.replace(pos, from.size(), to); + pos += to.size(); + } + return res; +} + + +string statusToString(int status) +{ + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) + return (format("failed with exit code %1%") % WEXITSTATUS(status)).str(); + else if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); +#if HAVE_STRSIGNAL + const char * description = strsignal(sig); + return (format("failed due to signal %1% (%2%)") % sig % description).str(); +#else + return (format("failed due to signal %1%") % sig).str(); +#endif + } + else + return "died abnormally"; + } else return "succeeded"; +} + + +bool statusOk(int status) +{ + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + + +bool hasPrefix(const string & s, const string & prefix) +{ + return s.compare(0, prefix.size(), prefix) == 0; +} + + +bool hasSuffix(const string & s, const string & suffix) +{ + return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix; +} + + +std::string toLower(const std::string & s) +{ + std::string r(s); + for (auto & c : r) + c = std::tolower(c); + return r; +} + + +std::string shellEscape(const std::string & s) +{ + std::string r = "'"; + for (auto & i : s) + if (i == '\'') r += "'\\''"; else r += i; + r += '\''; + return r; +} + + +void ignoreException() +{ + try { + throw; + } catch (std::exception & e) { + printError(format("error (ignored): %1%") % e.what()); + } +} + + +std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width) +{ + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; + } + + if (!filterAll && last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; + } + } + + else if (*i == '\r') + // do nothing for now + i++; + + else { + t += *i++; w++; + } + } + + return t; +} + + +static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + +string base64Encode(const string & s) +{ + string res; + int data = 0, nbits = 0; + + for (char c : s) { + data = data << 8 | (unsigned char) c; + nbits += 8; + while (nbits >= 6) { + nbits -= 6; + res.push_back(base64Chars[data >> nbits & 0x3f]); + } + } + + if (nbits) res.push_back(base64Chars[data << (6 - nbits) & 0x3f]); + while (res.size() % 4) res.push_back('='); + + return res; +} + + +string base64Decode(const string & s) +{ + bool init = false; + char decode[256]; + if (!init) { + // FIXME: not thread-safe. + memset(decode, -1, sizeof(decode)); + for (int i = 0; i < 64; i++) + decode[(int) base64Chars[i]] = i; + init = true; + } + + string res; + unsigned int d = 0, bits = 0; + + for (char c : s) { + if (c == '=') break; + if (c == '\n') continue; + + char digit = decode[(unsigned char) c]; + if (digit == -1) + throw Error("invalid character in Base64 string"); + + bits += 6; + d = d << 6 | digit; + if (bits >= 8) { + res.push_back(d >> (bits - 8) & 0xff); + bits -= 8; + } + } + + return res; +} + + +void callFailure(const std::function<void(std::exception_ptr exc)> & failure, std::exception_ptr exc) +{ + try { + failure(exc); + } catch (std::exception & e) { + printError(format("uncaught exception: %s") % e.what()); + abort(); + } +} + + +static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}}; + + +static void updateWindowSize() +{ + struct winsize ws; + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { + auto windowSize_(windowSize.lock()); + windowSize_->first = ws.ws_row; + windowSize_->second = ws.ws_col; + } +} + + +std::pair<unsigned short, unsigned short> getWindowSize() +{ + return *windowSize.lock(); +} + + +static Sync<std::list<std::function<void()>>> _interruptCallbacks; + +static void signalHandlerThread(sigset_t set) +{ + while (true) { + int signal = 0; + sigwait(&set, &signal); + + if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) + triggerInterrupt(); + + else if (signal == SIGWINCH) { + updateWindowSize(); + } + } +} + +void triggerInterrupt() +{ + _isInterrupted = true; + + { + auto interruptCallbacks(_interruptCallbacks.lock()); + for (auto & callback : *interruptCallbacks) { + try { + callback(); + } catch (...) { + ignoreException(); + } + } + } +} + +static sigset_t savedSignalMask; + +void startSignalHandlerThread() +{ + updateWindowSize(); + + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("quering signal mask"); + + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGPIPE); + sigaddset(&set, SIGWINCH); + if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) + throw SysError("blocking signals"); + + std::thread(signalHandlerThread, set).detach(); +} + +void restoreSignals() +{ + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); +} + +/* RAII helper to automatically deregister a callback. */ +struct InterruptCallbackImpl : InterruptCallback +{ + std::list<std::function<void()>>::iterator it; + ~InterruptCallbackImpl() override + { + _interruptCallbacks.lock()->erase(it); + } +}; + +std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()> callback) +{ + auto interruptCallbacks(_interruptCallbacks.lock()); + interruptCallbacks->push_back(callback); + + auto res = std::make_unique<InterruptCallbackImpl>(); + res->it = interruptCallbacks->end(); + res->it--; + + return std::unique_ptr<InterruptCallback>(res.release()); +} + +} diff --git a/third_party/nix/src/libutil/util.hh b/third_party/nix/src/libutil/util.hh new file mode 100644 index 000000000000..f057fdb2c041 --- /dev/null +++ b/third_party/nix/src/libutil/util.hh @@ -0,0 +1,542 @@ +#pragma once + +#include "types.hh" +#include "logging.hh" + +#include <sys/types.h> +#include <sys/stat.h> +#include <dirent.h> +#include <unistd.h> +#include <signal.h> + +#include <functional> +#include <limits> +#include <cstdio> +#include <map> +#include <sstream> +#include <optional> +#include <future> + +#ifndef HAVE_STRUCT_DIRENT_D_TYPE +#define DT_UNKNOWN 0 +#define DT_REG 1 +#define DT_LNK 2 +#define DT_DIR 3 +#endif + +namespace nix { + +struct Sink; +struct Source; + + +/* The system for which Nix is compiled. */ +extern const std::string nativeSystem; + + +/* Return an environment variable. */ +string getEnv(const string & key, const string & def = ""); + +/* Get the entire environment. */ +std::map<std::string, std::string> getEnv(); + +/* Clear the environment. */ +void clearEnv(); + +/* Return an absolutized path, resolving paths relative to the + specified directory, or the current directory otherwise. The path + is also canonicalised. */ +Path absPath(Path path, Path dir = ""); + +/* Canonicalise a path by removing all `.' or `..' components and + double or trailing slashes. Optionally resolves all symlink + components such that each component of the resulting path is *not* + a symbolic link. */ +Path canonPath(const Path & path, bool resolveSymlinks = false); + +/* Return the directory part of the given canonical path, i.e., + everything before the final `/'. If the path is the root or an + immediate child thereof (e.g., `/foo'), this means an empty string + is returned. */ +Path dirOf(const Path & path); + +/* Return the base name of the given canonical path, i.e., everything + following the final `/'. */ +string baseNameOf(const Path & path); + +/* Check whether 'path' is a descendant of 'dir'. */ +bool isInDir(const Path & path, const Path & dir); + +/* Check whether 'path' is equal to 'dir' or a descendant of 'dir'. */ +bool isDirOrInDir(const Path & path, const Path & dir); + +/* Get status of `path'. */ +struct stat lstat(const Path & path); + +/* Return true iff the given path exists. */ +bool pathExists(const Path & path); + +/* Read the contents (target) of a symbolic link. The result is not + in any way canonicalised. */ +Path readLink(const Path & path); + +bool isLink(const Path & path); + +/* Read the contents of a directory. The entries `.' and `..' are + removed. */ +struct DirEntry +{ + string name; + ino_t ino; + unsigned char type; // one of DT_* + DirEntry(const string & name, ino_t ino, unsigned char type) + : name(name), ino(ino), type(type) { } +}; + +typedef vector<DirEntry> DirEntries; + +DirEntries readDirectory(const Path & path); + +unsigned char getFileType(const Path & path); + +/* Read the contents of a file into a string. */ +string readFile(int fd); +string readFile(const Path & path, bool drain = false); +void readFile(const Path & path, Sink & sink); + +/* Write a string to a file. */ +void writeFile(const Path & path, const string & s, mode_t mode = 0666); + +void writeFile(const Path & path, Source & source, mode_t mode = 0666); + +/* Read a line from a file descriptor. */ +string readLine(int fd); + +/* Write a line to a file descriptor. */ +void writeLine(int fd, string s); + +/* Delete a path; i.e., in the case of a directory, it is deleted + recursively. It's not an error if the path does not exist. The + second variant returns the number of bytes and blocks freed. */ +void deletePath(const Path & path); + +void deletePath(const Path & path, unsigned long long & bytesFreed); + +/* Create a temporary directory. */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +std::string getUserName(); + +/* Return $HOME or the user's home directory from /etc/passwd. */ +Path getHome(); + +/* Return $XDG_CACHE_HOME or $HOME/.cache. */ +Path getCacheDir(); + +/* Return $XDG_CONFIG_HOME or $HOME/.config. */ +Path getConfigDir(); + +/* Return the directories to search for user configuration files */ +std::vector<Path> getConfigDirs(); + +/* Return $XDG_DATA_HOME or $HOME/.local/share. */ +Path getDataDir(); + +/* Create a directory and all its parents, if necessary. Returns the + list of created directories, in order of creation. */ +Paths createDirs(const Path & path); + +/* Create a symlink. */ +void createSymlink(const Path & target, const Path & link); + +/* Atomically create or replace a symlink. */ +void replaceSymlink(const Path & target, const Path & link); + + +/* Wrappers arount read()/write() that read/write exactly the + requested number of bytes. */ +void readFull(int fd, unsigned char * buf, size_t count); +void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true); +void writeFull(int fd, const string & s, bool allowInterrupts = true); + +MakeError(EndOfFile, Error) + + +/* Read a file descriptor until EOF occurs. */ +string drainFD(int fd, bool block = true); + +void drainFD(int fd, Sink & sink, bool block = true); + + +/* Automatic cleanup of resources. */ + + +class AutoDelete +{ + Path path; + bool del; + bool recursive; +public: + AutoDelete(); + AutoDelete(const Path & p, bool recursive = true); + ~AutoDelete(); + void cancel(); + void reset(const Path & p, bool recursive = true); + operator Path() const { return path; } +}; + + +class AutoCloseFD +{ + int fd; + void close(); +public: + AutoCloseFD(); + AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD & fd) = delete; + AutoCloseFD(AutoCloseFD&& fd); + ~AutoCloseFD(); + AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; + AutoCloseFD& operator =(AutoCloseFD&& fd); + int get() const; + explicit operator bool() const; + int release(); +}; + + +class Pipe +{ +public: + AutoCloseFD readSide, writeSide; + void create(); +}; + + +struct DIRDeleter +{ + void operator()(DIR * dir) const { + closedir(dir); + } +}; + +typedef std::unique_ptr<DIR, DIRDeleter> AutoCloseDir; + + +class Pid +{ + pid_t pid = -1; + bool separatePG = false; + int killSignal = SIGKILL; +public: + Pid(); + Pid(pid_t pid); + ~Pid(); + void operator =(pid_t pid); + operator pid_t(); + int kill(); + int wait(); + + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); + pid_t release(); +}; + + +/* Kill all processes running under the specified uid by sending them + a SIGKILL. */ +void killUser(uid_t uid); + + +/* Fork a process that runs the given function, and return the child + pid to the caller. */ +struct ProcessOptions +{ + string errorPrefix = "error: "; + bool dieWithParent = true; + bool runExitHandlers = false; + bool allowVfork = true; +}; + +pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions()); + + +/* Run a program and return its stdout in a string (i.e., like the + shell backtick operator). */ +string runProgram(Path program, bool searchPath = false, + const Strings & args = Strings(), + const std::optional<std::string> & input = {}); + +struct RunOptions +{ + std::optional<uid_t> uid; + std::optional<uid_t> gid; + std::optional<Path> chdir; + std::optional<std::map<std::string, std::string>> environment; + Path program; + bool searchPath = true; + Strings args; + std::optional<std::string> input; + Source * standardIn = nullptr; + Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; + bool _killStderr = false; + + RunOptions(const Path & program, const Strings & args) + : program(program), args(args) { }; + + RunOptions & killStderr(bool v) { _killStderr = true; return *this; } +}; + +std::pair<int, std::string> runProgram(const RunOptions & options); + +void runProgram2(const RunOptions & options); + + +class ExecError : public Error +{ +public: + int status; + + template<typename... Args> + ExecError(int status, Args... args) + : Error(args...), status(status) + { } +}; + +/* Convert a list of strings to a null-terminated vector of char + *'s. The result must not be accessed beyond the lifetime of the + list of strings. */ +std::vector<char *> stringsToCharPtrs(const Strings & ss); + +/* Close all file descriptors except those listed in the given set. + Good practice in child processes. */ +void closeMostFDs(const set<int> & exceptions); + +/* Set the close-on-exec flag for the given file descriptor. */ +void closeOnExec(int fd); + + +/* User interruption. */ + +extern bool _isInterrupted; + +extern thread_local std::function<bool()> interruptCheck; + +void setInterruptThrown(); + +void _interrupted(); + +void inline checkInterrupt() +{ + if (_isInterrupted || (interruptCheck && interruptCheck())) + _interrupted(); +} + +MakeError(Interrupted, BaseError) + + +MakeError(FormatError, Error) + + +/* String tokenizer. */ +template<class C> C tokenizeString(const string & s, const string & separators = " \t\n\r"); + + +/* Concatenate the given strings with a separator between the + elements. */ +string concatStringsSep(const string & sep, const Strings & ss); +string concatStringsSep(const string & sep, const StringSet & ss); + + +/* Remove trailing whitespace from a string. */ +string chomp(const string & s); + + +/* Remove whitespace from the start and end of a string. */ +string trim(const string & s, const string & whitespace = " \n\r\t"); + + +/* Replace all occurrences of a string inside another string. */ +string replaceStrings(const std::string & s, + const std::string & from, const std::string & to); + + +/* Convert the exit status of a child as returned by wait() into an + error string. */ +string statusToString(int status); + +bool statusOk(int status); + + +/* Parse a string into an integer. */ +template<class N> bool string2Int(const string & s, N & n) +{ + if (string(s, 0, 1) == "-" && !std::numeric_limits<N>::is_signed) + return false; + std::istringstream str(s); + str >> n; + return str && str.get() == EOF; +} + +/* Parse a string into a float. */ +template<class N> bool string2Float(const string & s, N & n) +{ + std::istringstream str(s); + str >> n; + return str && str.get() == EOF; +} + + +/* Return true iff `s' starts with `prefix'. */ +bool hasPrefix(const string & s, const string & prefix); + + +/* Return true iff `s' ends in `suffix'. */ +bool hasSuffix(const string & s, const string & suffix); + + +/* Convert a string to lower case. */ +std::string toLower(const std::string & s); + + +/* Escape a string as a shell word. */ +std::string shellEscape(const std::string & s); + + +/* Exception handling in destructors: print an error message, then + ignore the exception. */ +void ignoreException(); + + +/* Some ANSI escape sequences. */ +#define ANSI_NORMAL "\e[0m" +#define ANSI_BOLD "\e[1m" +#define ANSI_FAINT "\e[2m" +#define ANSI_RED "\e[31;1m" +#define ANSI_GREEN "\e[32;1m" +#define ANSI_BLUE "\e[34;1m" + + +/* Truncate a string to 'width' printable characters. If 'filterAll' + is true, all ANSI escape sequences are filtered out. Otherwise, + some escape sequences (such as colour setting) are copied but not + included in the character count. Also, tabs are expanded to + spaces. */ +std::string filterANSIEscapes(const std::string & s, + bool filterAll = false, + unsigned int width = std::numeric_limits<unsigned int>::max()); + + +/* Base64 encoding/decoding. */ +string base64Encode(const string & s); +string base64Decode(const string & s); + + +/* Get a value for the specified key from an associate container, or a + default value if the key doesn't exist. */ +template <class T> +string get(const T & map, const string & key, const string & def = "") +{ + auto i = map.find(key); + return i == map.end() ? def : i->second; +} + + +/* A callback is a wrapper around a lambda that accepts a valid of + type T or an exception. (We abuse std::future<T> to pass the value or + exception.) */ +template<typename T> +class Callback +{ + std::function<void(std::future<T>)> fun; + std::atomic_flag done = ATOMIC_FLAG_INIT; + +public: + + Callback(std::function<void(std::future<T>)> fun) : fun(fun) { } + + Callback(Callback && callback) : fun(std::move(callback.fun)) + { + auto prev = callback.done.test_and_set(); + if (prev) done.test_and_set(); + } + + void operator()(T && t) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); + std::promise<T> promise; + promise.set_value(std::move(t)); + fun(promise.get_future()); + } + + void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); + std::promise<T> promise; + promise.set_exception(exc); + fun(promise.get_future()); + } +}; + + +/* Start a thread that handles various signals. Also block those signals + on the current thread (and thus any threads created by it). */ +void startSignalHandlerThread(); + +/* Restore default signal handling. */ +void restoreSignals(); + +struct InterruptCallback +{ + virtual ~InterruptCallback() { }; +}; + +/* Register a function that gets called on SIGINT (in a non-signal + context). */ +std::unique_ptr<InterruptCallback> createInterruptCallback( + std::function<void()> callback); + +void triggerInterrupt(); + +/* A RAII class that causes the current thread to receive SIGUSR1 when + the signal handler thread receives SIGINT. That is, this allows + SIGINT to be multiplexed to multiple threads. */ +struct ReceiveInterrupts +{ + pthread_t target; + std::unique_ptr<InterruptCallback> callback; + + ReceiveInterrupts() + : target(pthread_self()) + , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) + { } +}; + + + +/* A RAII helper that increments a counter on construction and + decrements it on destruction. */ +template<typename T> +struct MaintainCount +{ + T & counter; + long delta; + MaintainCount(T & counter, long delta = 1) : counter(counter), delta(delta) { counter += delta; } + ~MaintainCount() { counter -= delta; } +}; + + +/* Return the number of rows and columns of the terminal. */ +std::pair<unsigned short, unsigned short> getWindowSize(); + + +/* Used in various places. */ +typedef std::function<bool(const Path & path)> PathFilter; + +extern PathFilter defaultPathFilter; + + +} diff --git a/third_party/nix/src/libutil/xml-writer.cc b/third_party/nix/src/libutil/xml-writer.cc new file mode 100644 index 000000000000..e5cc2e9fc719 --- /dev/null +++ b/third_party/nix/src/libutil/xml-writer.cc @@ -0,0 +1,94 @@ +#include <assert.h> + +#include "xml-writer.hh" + + +namespace nix { + + +XMLWriter::XMLWriter(bool indent, std::ostream & output) + : output(output), indent(indent) +{ + output << "<?xml version='1.0' encoding='utf-8'?>" << std::endl; + closed = false; +} + + +XMLWriter::~XMLWriter() +{ + close(); +} + + +void XMLWriter::close() +{ + if (closed) return; + while (!pendingElems.empty()) closeElement(); + closed = true; +} + + +void XMLWriter::indent_(size_t depth) +{ + if (!indent) return; + output << string(depth * 2, ' '); +} + + +void XMLWriter::openElement(const string & name, + const XMLAttrs & attrs) +{ + assert(!closed); + indent_(pendingElems.size()); + output << "<" << name; + writeAttrs(attrs); + output << ">"; + if (indent) output << std::endl; + pendingElems.push_back(name); +} + + +void XMLWriter::closeElement() +{ + assert(!pendingElems.empty()); + indent_(pendingElems.size() - 1); + output << "</" << pendingElems.back() << ">"; + if (indent) output << std::endl; + pendingElems.pop_back(); + if (pendingElems.empty()) closed = true; +} + + +void XMLWriter::writeEmptyElement(const string & name, + const XMLAttrs & attrs) +{ + assert(!closed); + indent_(pendingElems.size()); + output << "<" << name; + writeAttrs(attrs); + output << " />"; + if (indent) output << std::endl; +} + + +void XMLWriter::writeAttrs(const XMLAttrs & attrs) +{ + for (auto & i : attrs) { + output << " " << i.first << "=\""; + for (size_t j = 0; j < i.second.size(); ++j) { + char c = i.second[j]; + if (c == '"') output << """; + else if (c == '<') output << "<"; + else if (c == '>') output << ">"; + else if (c == '&') output << "&"; + /* Escape newlines to prevent attribute normalisation (see + XML spec, section 3.3.3. */ + else if (c == '\n') output << "
"; + else output << c; + } + output << "\""; + } +} + + +} diff --git a/third_party/nix/src/libutil/xml-writer.hh b/third_party/nix/src/libutil/xml-writer.hh new file mode 100644 index 000000000000..b98b445265a2 --- /dev/null +++ b/third_party/nix/src/libutil/xml-writer.hh @@ -0,0 +1,69 @@ +#pragma once + +#include <iostream> +#include <string> +#include <list> +#include <map> + + +namespace nix { + +using std::string; +using std::map; +using std::list; + + +typedef map<string, string> XMLAttrs; + + +class XMLWriter +{ +private: + + std::ostream & output; + + bool indent; + bool closed; + + list<string> pendingElems; + +public: + + XMLWriter(bool indent, std::ostream & output); + ~XMLWriter(); + + void close(); + + void openElement(const string & name, + const XMLAttrs & attrs = XMLAttrs()); + void closeElement(); + + void writeEmptyElement(const string & name, + const XMLAttrs & attrs = XMLAttrs()); + +private: + void writeAttrs(const XMLAttrs & attrs); + + void indent_(size_t depth); +}; + + +class XMLOpenElement +{ +private: + XMLWriter & writer; +public: + XMLOpenElement(XMLWriter & writer, const string & name, + const XMLAttrs & attrs = XMLAttrs()) + : writer(writer) + { + writer.openElement(name, attrs); + } + ~XMLOpenElement() + { + writer.closeElement(); + } +}; + + +} |