From 0db9e6cd1af299f7d65e0b99019f0ccdbb1aaf3f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 9 Feb 2016 21:07:48 +0100 Subject: New command line parsing infrastructure --- src/libmain/common-args.cc | 37 ++++++++++ src/libmain/common-args.hh | 22 ++++++ src/libmain/shared.cc | 148 ++++++++++++++++++------------------- src/libmain/shared.hh | 3 +- src/libutil/args.cc | 179 +++++++++++++++++++++++++++++++++++++++++++++ src/libutil/args.hh | 160 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 470 insertions(+), 79 deletions(-) create mode 100644 src/libmain/common-args.cc create mode 100644 src/libmain/common-args.hh create mode 100644 src/libutil/args.cc create mode 100644 src/libutil/args.hh diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc new file mode 100644 index 000000000000..54dbb1da3ca8 --- /dev/null +++ b/src/libmain/common-args.cc @@ -0,0 +1,37 @@ +#include "common-args.hh" +#include "globals.hh" + +namespace nix { + +MixCommonArgs::MixCommonArgs(const string & programName) +{ + mkFlag('v', "verbose", "increase verbosity level", []() { + verbosity = (Verbosity) (verbosity + 1); + }); + + mkFlag(0, "quiet", "decrease verbosity level", []() { + verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; + }); + + mkFlag(0, "debug", "enable debug output", []() { + verbosity = lvlDebug; + }); + + mkFlag1(0, "log-type", "type", "set logging format ('pretty', 'flat', 'systemd')", + [](std::string s) { + if (s == "pretty") logType = ltPretty; + else if (s == "escapes") logType = ltEscapes; + else if (s == "flat") logType = ltFlat; + else if (s == "systemd") logType = ltSystemd; + else throw UsageError("unknown log type"); + }); + + mkFlag(0, "option", {"name", "value"}, "set a Nix configuration option (overriding nix.conf)", 2, + [](Strings ss) { + auto name = ss.front(); ss.pop_front(); + auto value = ss.front(); + settings.set(name, value); + }); +} + +} diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh new file mode 100644 index 000000000000..62657e0d0218 --- /dev/null +++ b/src/libmain/common-args.hh @@ -0,0 +1,22 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +struct MixCommonArgs : virtual Args +{ + MixCommonArgs(const string & programName); +}; + +struct MixDryRun : virtual Args +{ + bool dryRun; + + MixDryRun() + { + mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun); + } +}; + +} diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 88ed52497fb9..bf6bf5b712e7 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,7 +1,8 @@ #include "config.h" -#include "shared.hh" +#include "common-args.hh" #include "globals.hh" +#include "shared.hh" #include "store-api.hh" #include "util.hh" @@ -84,16 +85,6 @@ void printMissing(ref store, const PathSet & willBuild, } -static void setLogType(string lt) -{ - if (lt == "pretty") logType = ltPretty; - else if (lt == "escapes") logType = ltEscapes; - else if (lt == "flat") logType = ltFlat; - else if (lt == "systemd") logType = ltSystemd; - else throw UsageError("unknown log type"); -} - - string getArg(const string & opt, Strings::iterator & i, const Strings::iterator & end) { @@ -164,77 +155,80 @@ void initNix() } -void parseCmdLine(int argc, char * * argv, - std::function parseArg) +struct LegacyArgs : public MixCommonArgs { - /* Put the arguments in a vector. */ - Strings args; - argc--; argv++; - while (argc--) args.push_back(*argv++); - - /* Process default options. */ - for (Strings::iterator i = args.begin(); i != args.end(); ++i) { - string arg = *i; - - /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f'). */ - if (arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) { - *i = (string) "-" + arg[1]; - auto next = i; ++next; - for (unsigned int j = 2; j < arg.length(); j++) - if (isalpha(arg[j])) - args.insert(next, (string) "-" + arg[j]); - else { - args.insert(next, string(arg, j)); - break; - } - arg = *i; - } + std::function parseArg; - if (arg == "--verbose" || arg == "-v") verbosity = (Verbosity) (verbosity + 1); - else if (arg == "--quiet") verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; - else if (arg == "--log-type") { - string s = getArg(arg, i, args.end()); - setLogType(s); - } - else if (arg == "--no-build-output" || arg == "-Q") - settings.buildVerbosity = lvlVomit; - else if (arg == "--print-build-trace") - settings.printBuildTrace = true; - else if (arg == "--keep-failed" || arg == "-K") - settings.keepFailed = true; - else if (arg == "--keep-going" || arg == "-k") - settings.keepGoing = true; - else if (arg == "--fallback") + LegacyArgs(const std::string & programName, + std::function parseArg) + : MixCommonArgs(programName), parseArg(parseArg) + { + mkFlag('Q', "no-build-output", "do not show build output", + &settings.buildVerbosity, lvlVomit); + + mkFlag(0, "print-build-trace", "emit special build trace message", + &settings.printBuildTrace); + + mkFlag('K', "keep-failed", "keep temporary directories of failed builds", + &settings.keepFailed); + + mkFlag('k', "keep-going", "keep going after a build fails", + &settings.keepGoing); + + mkFlag(0, "fallback", "build from source if substitution fails", []() { settings.set("build-fallback", "true"); - else if (arg == "--max-jobs" || arg == "-j") - settings.set("build-max-jobs", getArg(arg, i, args.end())); - else if (arg == "--cores") - settings.set("build-cores", getArg(arg, i, args.end())); - else if (arg == "--readonly-mode") - settings.readOnlyMode = true; - else if (arg == "--max-silent-time") - settings.set("build-max-silent-time", getArg(arg, i, args.end())); - else if (arg == "--timeout") - settings.set("build-timeout", getArg(arg, i, args.end())); - else if (arg == "--no-build-hook") - settings.useBuildHook = false; - else if (arg == "--show-trace") - settings.showTrace = true; - else if (arg == "--no-gc-warning") - gcWarning = false; - else if (arg == "--option") { - ++i; if (i == args.end()) throw UsageError("‘--option’ requires two arguments"); - string name = *i; - ++i; if (i == args.end()) throw UsageError("‘--option’ requires two arguments"); - string value = *i; - settings.set(name, value); - } - else { - if (!parseArg(i, args.end())) - throw UsageError(format("unrecognised option ‘%1%’") % *i); - } + }); + + auto intSettingAlias = [&](char shortName, const std::string & longName, + const std::string & description, const std::string & dest) { + mkFlag(shortName, longName, description, [=](unsigned int n) { + settings.set(dest, std::to_string(n)); + }); + }; + + intSettingAlias('j', "max-jobs", "maximum number of parallel builds", "build-max-jobs"); + intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "build-cores"); + intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "build-max-silent-time"); + intSettingAlias(0, "timeout", "number of seconds before a build is killed", "build-timeout"); + + mkFlag(0, "readonly-mode", "do not write to the Nix store", + &settings.readOnlyMode); + + mkFlag(0, "no-build-hook", "disable use of the build hook mechanism", + &settings.useBuildHook, false); + + mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors", + &settings.showTrace); + + mkFlag(0, "no-gc-warning", "disable warning about not using ‘--add-root’", + &gcWarning, false); } + bool processFlag(Strings::iterator & pos, Strings::iterator end) override + { + if (MixCommonArgs::processFlag(pos, end)) return true; + bool res = parseArg(pos, end); + if (res) ++pos; + return res; + } + + bool processArgs(const Strings & args, bool finish) override + { + if (args.empty()) return true; + assert(args.size() == 1); + Strings ss(args); + auto pos = ss.begin(); + if (!parseArg(pos, ss.end())) + throw UsageError(format("unexpected argument ‘%1%’") % args.front()); + return true; + } +}; + + +void parseCmdLine(int argc, char * * argv, + std::function parseArg) +{ + LegacyArgs(baseNameOf(argv[0]), parseArg).parseCmdline(argvToStrings(argc, argv)); settings.update(); } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index e4b984f067e4..20744be0ab9b 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,6 +1,7 @@ #pragma once #include "util.hh" +#include "args.hh" #include @@ -9,8 +10,6 @@ namespace nix { -MakeError(UsageError, nix::Error); - class Exit : public std::exception { public: diff --git a/src/libutil/args.cc b/src/libutil/args.cc new file mode 100644 index 000000000000..6e4b82a279ce --- /dev/null +++ b/src/libutil/args.cc @@ -0,0 +1,179 @@ +#include "args.hh" +#include "hash.hh" + +namespace nix { + +void Args::parseCmdline(const Strings & _cmdline) +{ + Strings pendingArgs; + bool dashDash = false; + + Strings cmdline(_cmdline); + + for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { + + auto arg = *pos; + + /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f', + `-j3` -> `-j 3`). */ + if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) { + *pos = (string) "-" + arg[1]; + auto next = pos; ++next; + for (unsigned int j = 2; j < arg.length(); j++) + if (isalpha(arg[j])) + cmdline.insert(next, (string) "-" + arg[j]); + else { + cmdline.insert(next, string(arg, j)); + break; + } + arg = *pos; + } + + if (!dashDash && arg == "--") { + dashDash = true; + ++pos; + } + else if (!dashDash && std::string(arg, 0, 1) == "-") { + if (!processFlag(pos, cmdline.end())) + throw UsageError(format("unrecognised flag ‘%1%’") % arg); + } + else { + pendingArgs.push_back(*pos++); + if (processArgs(pendingArgs, false)) + pendingArgs.clear(); + } + } + + processArgs(pendingArgs, true); +} + +void Args::printHelp(const string & programName, std::ostream & out) +{ + std::cout << "Usage: " << programName << " ..."; + for (auto & exp : expectedArgs) { + std::cout << renderLabels({exp.label}); + // FIXME: handle arity > 1 + if (exp.arity == 0) std::cout << "..."; + } + std::cout << "\n"; + + auto s = description(); + if (s != "") + std::cout << "\nSummary: " << s << ".\n"; + + if (longFlags.size()) { + std::cout << "\n"; + std::cout << "Flags:\n"; + printFlags(out); + } +} + +void Args::printFlags(std::ostream & out) +{ + Table2 table; + for (auto & flags : longFlags) + table.push_back(std::make_pair( + "--" + flags.first + renderLabels(flags.second.labels), + flags.second.description)); + printTable(out, table); +} + +bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + assert(pos != end); + + auto process = [&](const std::string & name, const Flag & flag) -> bool { + ++pos; + Strings args; + for (size_t n = 0 ; n < flag.arity; ++n) { + if (pos == end) + throw UsageError(format("flag ‘%1%’ requires %2% argument(s)") + % name % flag.arity); + args.push_back(*pos++); + } + flag.handler(args); + return true; + }; + + if (string(*pos, 0, 2) == "--") { + auto i = longFlags.find(string(*pos, 2)); + if (i == longFlags.end()) return false; + return process("--" + i->first, i->second); + } + + if (string(*pos, 0, 1) == "-" && pos->size() == 2) { + auto c = (*pos)[1]; + auto i = shortFlags.find(c); + if (i == shortFlags.end()) return false; + return process(std::string("-") + c, i->second); + } + + return false; +} + +bool Args::processArgs(const Strings & args, bool finish) +{ + if (expectedArgs.empty()) { + if (!args.empty()) + throw UsageError(format("unexpected argument ‘%1%’") % args.front()); + return true; + } + + auto & exp = expectedArgs.front(); + + bool res = false; + + if ((exp.arity == 0 && finish) || + (exp.arity > 0 && args.size() == exp.arity)) + { + exp.handler(args); + expectedArgs.pop_front(); + res = true; + } + + if (finish && !expectedArgs.empty()) + throw UsageError("more arguments are required"); + + return res; +} + +void Args::mkHashTypeFlag(const std::string & name, HashType * ht) +{ + mkFlag1(0, name, "TYPE", "hash algorithm (‘md5’, ‘sha1’, ‘sha256’, or ‘sha512’)", [=](std::string s) { + *ht = parseHashType(s); + if (*ht == htUnknown) + throw UsageError(format("unknown hash type ‘%1%’") % s); + }); +} + +Strings argvToStrings(int argc, char * * argv) +{ + Strings args; + argc--; argv++; + while (argc--) args.push_back(*argv++); + return args; +} + +std::string renderLabels(const Strings & labels) +{ + std::string res; + for (auto label : labels) { + for (auto & c : label) c = std::toupper(c); + res += " <" + label + ">"; + } + return res; +} + +void printTable(std::ostream & out, const Table2 & table) +{ + size_t max = 0; + for (auto & row : table) + max = std::max(max, row.first.size()); + for (auto & row : table) { + out << " " << row.first + << std::string(max - row.first.size() + 2, ' ') + << row.second << "\n"; + } +} + +} diff --git a/src/libutil/args.hh b/src/libutil/args.hh new file mode 100644 index 000000000000..c61a2b02a7ac --- /dev/null +++ b/src/libutil/args.hh @@ -0,0 +1,160 @@ +#pragma once + +#include +#include +#include + +#include "util.hh" + +namespace nix { + +MakeError(UsageError, nix::Error); + +enum HashType : char; + +class Args +{ +public: + + /* Parse the command line, throwing a UsageError if something goes + wrong. */ + void parseCmdline(const Strings & cmdline); + + virtual void printHelp(const string & programName, std::ostream & out); + + virtual std::string description() { return ""; } + +protected: + + /* Flags. */ + struct Flag + { + std::string description; + Strings labels; + size_t arity; + std::function handler; + }; + + std::map longFlags; + std::map shortFlags; + + virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); + + void printFlags(std::ostream & out); + + /* Positional arguments. */ + struct ExpectedArg + { + std::string label; + size_t arity; // 0 = any + std::function handler; + }; + + std::list expectedArgs; + + virtual bool processArgs(const Strings & args, bool finish); + +public: + + /* Helper functions for constructing flags / positional + arguments. */ + + void mkFlag(char shortName, const std::string & longName, + const Strings & labels, const std::string & description, + size_t arity, std::function handler) + { + auto flag = Flag{description, labels, arity, handler}; + if (shortName) shortFlags[shortName] = flag; + longFlags[longName] = flag; + } + + void mkFlag(char shortName, const std::string & longName, + const std::string & description, std::function fun) + { + mkFlag(shortName, longName, {}, description, 0, std::bind(fun)); + } + + void mkFlag1(char shortName, const std::string & longName, + const std::string & label, const std::string & description, + std::function fun) + { + mkFlag(shortName, longName, {label}, description, 1, [=](Strings ss) { + fun(ss.front()); + }); + } + + void mkFlag(char shortName, const std::string & name, + const std::string & description, bool * dest) + { + mkFlag(0, name, description, dest, true); + } + + void mkFlag(char shortName, const std::string & longName, + const std::string & label, const std::string & description, + string * dest) + { + mkFlag1(shortName, longName, label, description, [=](std::string s) { + *dest = s; + }); + } + + void mkHashTypeFlag(const std::string & name, HashType * ht); + + template + void mkFlag(char shortName, const std::string & longName, const std::string & description, + T * dest, const T & value) + { + mkFlag(shortName, longName, {}, description, 0, [=](Strings ss) { + *dest = value; + }); + } + + template + void mkIntFlag(char shortName, const std::string & longName, + const std::string & description, I * dest) + { + mkFlag(shortName, longName, description, [=](I n) { + *dest = n; + }); + } + + template + void mkFlag(char shortName, const std::string & longName, + const std::string & description, std::function fun) + { + mkFlag(shortName, longName, {"N"}, description, 1, [=](Strings ss) { + I n; + if (!string2Int(ss.front(), n)) + throw UsageError(format("flag ‘--%1%’ requires a integer argument") % longName); + fun(n); + }); + } + + /* Expect a string argument. */ + void expectArg(const std::string & label, string * dest) + { + expectedArgs.push_back(ExpectedArg{label, 1, [=](Strings ss) { + *dest = ss.front(); + }}); + } + + /* Expect 0 or more arguments. */ + void expectArgs(const std::string & label, Strings * dest) + { + expectedArgs.push_back(ExpectedArg{label, 0, [=](Strings ss) { + *dest = ss; + }}); + } +}; + +Strings argvToStrings(int argc, char * * argv); + +/* Helper function for rendering argument labels. */ +std::string renderLabels(const Strings & labels); + +/* Helper function for printing 2-column tables. */ +typedef std::vector> Table2; + +void printTable(std::ostream & out, const Table2 & table); + +} -- cgit 1.4.1 From cd2196b08981a96cf607ad4a8f2f0dfa8cdf2add Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 9 Feb 2016 21:28:29 +0100 Subject: Start of new Nix command-line interface --- .gitignore | 3 - Makefile | 2 +- src/libmain/common-args.cc | 1 + src/libmain/common-args.hh | 1 + src/libutil/args.hh | 2 + src/nix-hash/local.mk | 7 --- src/nix-hash/nix-hash.cc | 63 -------------------- src/nix/command.cc | 65 +++++++++++++++++++++ src/nix/command.hh | 59 +++++++++++++++++++ src/nix/hash.cc | 140 +++++++++++++++++++++++++++++++++++++++++++++ src/nix/legacy.cc | 7 +++ src/nix/legacy.hh | 22 +++++++ src/nix/local.mk | 9 +++ src/nix/main.cc | 55 ++++++++++++++++++ 14 files changed, 362 insertions(+), 74 deletions(-) delete mode 100644 src/nix-hash/local.mk delete mode 100644 src/nix-hash/nix-hash.cc create mode 100644 src/nix/command.cc create mode 100644 src/nix/command.hh create mode 100644 src/nix/hash.cc create mode 100644 src/nix/legacy.cc create mode 100644 src/nix/legacy.hh create mode 100644 src/nix/local.mk create mode 100644 src/nix/main.cc diff --git a/.gitignore b/.gitignore index 94067256b8bc..de8e9354fbbe 100644 --- a/.gitignore +++ b/.gitignore @@ -73,9 +73,6 @@ Makefile.config # /src/nix-env/ /src/nix-env/nix-env -# /src/nix-hash/ -/src/nix-hash/nix-hash - # /src/nix-instantiate/ /src/nix-instantiate/nix-instantiate diff --git a/Makefile b/Makefile index 3a204de888cc..39870af75104 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ makefiles = \ src/libstore/local.mk \ src/libmain/local.mk \ src/libexpr/local.mk \ - src/nix-hash/local.mk \ + src/nix/local.mk \ src/nix-store/local.mk \ src/nix-instantiate/local.mk \ src/nix-env/local.mk \ diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 54dbb1da3ca8..9219f380c74f 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -4,6 +4,7 @@ namespace nix { MixCommonArgs::MixCommonArgs(const string & programName) + : programName(programName) { mkFlag('v', "verbose", "increase verbosity level", []() { verbosity = (Verbosity) (verbosity + 1); diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index 62657e0d0218..2c0d71edd815 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -6,6 +6,7 @@ namespace nix { struct MixCommonArgs : virtual Args { + string programName; MixCommonArgs(const string & programName); }; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index c61a2b02a7ac..0c7729d0b6b1 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -145,6 +145,8 @@ public: *dest = ss; }}); } + + friend class MultiCommand; }; Strings argvToStrings(int argc, char * * argv); diff --git a/src/nix-hash/local.mk b/src/nix-hash/local.mk deleted file mode 100644 index 7c290ca8466e..000000000000 --- a/src/nix-hash/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-hash - -nix-hash_DIR := $(d) - -nix-hash_SOURCES := $(d)/nix-hash.cc - -nix-hash_LIBS = libmain libstore libutil libformat diff --git a/src/nix-hash/nix-hash.cc b/src/nix-hash/nix-hash.cc deleted file mode 100644 index 8035162aea37..000000000000 --- a/src/nix-hash/nix-hash.cc +++ /dev/null @@ -1,63 +0,0 @@ -#include "hash.hh" -#include "shared.hh" - -#include - -using namespace nix; - - -int main(int argc, char * * argv) -{ - HashType ht = htMD5; - bool flat = false; - bool base32 = false; - bool truncate = false; - enum { opHash, opTo32, opTo16 } op = opHash; - - Strings ss; - - return handleExceptions(argv[0], [&]() { - initNix(); - - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") - showManPage("nix-hash"); - else if (*arg == "--version") - printVersion("nix-hash"); - else if (*arg == "--flat") flat = true; - else if (*arg == "--base32") base32 = true; - else if (*arg == "--truncate") truncate = true; - else if (*arg == "--type") { - string s = getArg(*arg, arg, end); - ht = parseHashType(s); - if (ht == htUnknown) - throw UsageError(format("unknown hash type ‘%1%’") % s); - } - else if (*arg == "--to-base16") op = opTo16; - else if (*arg == "--to-base32") op = opTo32; - else if (*arg != "" && arg->at(0) == '-') - return false; - else - ss.push_back(*arg); - return true; - }); - - if (op == opHash) { - for (auto & i : ss) { - Hash h = flat ? hashFile(ht, i) : hashPath(ht, i).first; - if (truncate && h.hashSize > 20) h = compressHash(h, 20); - std::cout << format("%1%\n") % - (base32 ? printHash32(h) : printHash(h)); - } - } - - else { - for (auto & i : ss) { - Hash h = parseHash16or32(ht, i); - std::cout << format("%1%\n") % - (op == opTo16 ? printHash(h) : printHash32(h)); - } - } - }); -} - diff --git a/src/nix/command.cc b/src/nix/command.cc new file mode 100644 index 000000000000..698863349334 --- /dev/null +++ b/src/nix/command.cc @@ -0,0 +1,65 @@ +#include "command.hh" +#include "store-api.hh" + +namespace nix { + +Commands * RegisterCommand::commands = 0; + +MultiCommand::MultiCommand(const Commands & _commands) + : commands(_commands) +{ + expectedArgs.push_back(ExpectedArg{"command", 1, [=](Strings ss) { + assert(!command); + auto i = commands.find(ss.front()); + if (i == commands.end()) + throw UsageError(format("‘%1%’ is not a recognised command") % ss.front()); + command = i->second; + }}); +} + +void MultiCommand::printHelp(const string & programName, std::ostream & out) +{ + if (command) { + command->printHelp(programName + " " + command->name(), out); + return; + } + + out << "Usage: " << programName << " ... ...\n"; + + out << "\n"; + out << "Common flags:\n"; + printFlags(out); + + out << "\n"; + out << "Available commands:\n"; + + Table2 table; + for (auto & command : commands) + table.push_back(std::make_pair(command.second->name(), command.second->description())); + printTable(out, table); + + out << "\n"; + out << "For full documentation, run ‘man " << programName << "’ or ‘man " << programName << "-’.\n"; +} + +bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + if (Args::processFlag(pos, end)) return true; + if (command && command->processFlag(pos, end)) return true; + return false; +} + +bool MultiCommand::processArgs(const Strings & args, bool finish) +{ + if (command) + return command->processArgs(args, finish); + else + return Args::processArgs(args, finish); +} + +void StoreCommand::run() +{ + run(openStore(reserveSpace)); +} + +} diff --git a/src/nix/command.hh b/src/nix/command.hh new file mode 100644 index 000000000000..a84721ccfa8c --- /dev/null +++ b/src/nix/command.hh @@ -0,0 +1,59 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +/* A command is an argument parser that can be executed by calling its + run() method. */ +struct Command : virtual Args +{ + virtual std::string name() = 0; + virtual void prepare() { }; + virtual void run() = 0; +}; + +class Store; + +/* A command that require a Nix store. */ +struct StoreCommand : virtual Command +{ + bool reserveSpace; + StoreCommand(bool reserveSpace = true) + : reserveSpace(reserveSpace) { }; + void run() override; + virtual void run(ref) = 0; +}; + +typedef std::map> Commands; + +/* An argument parser that supports multiple subcommands, + i.e. ‘ ’. */ +struct MultiCommand : virtual Args +{ + Commands commands; + + std::shared_ptr command; + + MultiCommand(const Commands & commands); + + void printHelp(const string & programName, std::ostream & out) override; + + bool processFlag(Strings::iterator & pos, Strings::iterator end) override; + + bool processArgs(const Strings & args, bool finish) override; +}; + +/* A helper class for registering commands globally. */ +struct RegisterCommand +{ + static Commands * commands; + + RegisterCommand(ref command) + { + if (!commands) commands = new Commands; + commands->emplace(command->name(), command); + } +}; + +} diff --git a/src/nix/hash.cc b/src/nix/hash.cc new file mode 100644 index 000000000000..fff945bd33d9 --- /dev/null +++ b/src/nix/hash.cc @@ -0,0 +1,140 @@ +#include "command.hh" +#include "hash.hh" +#include "legacy.hh" +#include "shared.hh" + +using namespace nix; + +struct CmdHash : Command +{ + enum Mode { mFile, mPath }; + Mode mode; + bool base32 = false; + bool truncate = false; + HashType ht = htSHA512; + Strings paths; + + CmdHash(Mode mode) : mode(mode) + { + mkFlag(0, "base32", "print hash in base-32", &base32); + mkFlag(0, "base16", "print hash in base-16", &base32, false); + mkHashTypeFlag("type", &ht); + expectArgs("paths", &paths); + } + + std::string name() override + { + return mode == mFile ? "hash-file" : "hash-path"; + } + + std::string description() override + { + return mode == mFile + ? "print cryptographic hash of a regular file" + : "print cryptographic hash of the NAR serialisation of a path"; + } + + void run() override + { + for (auto path : paths) { + Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first; + if (truncate && h.hashSize > 20) h = compressHash(h, 20); + std::cout << format("%1%\n") % + (base32 ? printHash32(h) : printHash(h)); + } + } +}; + +static RegisterCommand r1(make_ref(CmdHash::mFile)); +static RegisterCommand r2(make_ref(CmdHash::mPath)); + +struct CmdToBase : Command +{ + bool toBase32; + HashType ht = htSHA512; + Strings args; + + CmdToBase(bool toBase32) : toBase32(toBase32) + { + mkHashTypeFlag("type", &ht); + expectArgs("strings", &args); + } + + std::string name() override + { + return toBase32 ? "to-base32" : "to-base16"; + } + + std::string description() override + { + return toBase32 + ? "convert a hash to base-32 representation" + : "convert a hash to base-32 representation"; + } + + void run() override + { + for (auto s : args) { + Hash h = parseHash16or32(ht, s); + std::cout << format("%1%\n") % + (toBase32 ? printHash32(h) : printHash(h)); + } + } +}; + +static RegisterCommand r3(make_ref(false)); +static RegisterCommand r4(make_ref(true)); + +/* Legacy nix-hash command. */ +static int compatNixHash(int argc, char * * argv) +{ + HashType ht = htMD5; + bool flat = false; + bool base32 = false; + bool truncate = false; + enum { opHash, opTo32, opTo16 } op = opHash; + Strings ss; + + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-hash"); + else if (*arg == "--version") + printVersion("nix-hash"); + else if (*arg == "--flat") flat = true; + else if (*arg == "--base32") base32 = true; + else if (*arg == "--truncate") truncate = true; + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + if (ht == htUnknown) + throw UsageError(format("unknown hash type ‘%1%’") % s); + } + else if (*arg == "--to-base16") op = opTo16; + else if (*arg == "--to-base32") op = opTo32; + else if (*arg != "" && arg->at(0) == '-') + return false; + else + ss.push_back(*arg); + return true; + }); + + if (op == opHash) { + CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath); + cmd.ht = ht; + cmd.base32 = base32; + cmd.truncate = truncate; + cmd.paths = ss; + cmd.run(); + } + + else { + CmdToBase cmd(op == opTo32); + cmd.args = ss; + cmd.ht = ht; + cmd.run(); + } + + return 0; +} + +static RegisterLegacyCommand s1("nix-hash", compatNixHash); diff --git a/src/nix/legacy.cc b/src/nix/legacy.cc new file mode 100644 index 000000000000..6df09ee37a5e --- /dev/null +++ b/src/nix/legacy.cc @@ -0,0 +1,7 @@ +#include "legacy.hh" + +namespace nix { + +RegisterLegacyCommand::Commands * RegisterLegacyCommand::commands = 0; + +} diff --git a/src/nix/legacy.hh b/src/nix/legacy.hh new file mode 100644 index 000000000000..b67b70eb5c85 --- /dev/null +++ b/src/nix/legacy.hh @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +namespace nix { + +typedef std::function MainFunction; + +struct RegisterLegacyCommand +{ + typedef std::map Commands; + static Commands * commands; + + RegisterLegacyCommand(const std::string & name, MainFunction fun) + { + if (!commands) commands = new Commands; + (*commands)[name] = fun; + } +}; + +} diff --git a/src/nix/local.mk b/src/nix/local.mk new file mode 100644 index 000000000000..f6e7073b6e7d --- /dev/null +++ b/src/nix/local.mk @@ -0,0 +1,9 @@ +programs += nix + +nix_DIR := $(d) + +nix_SOURCES := $(wildcard $(d)/*.cc) + +nix_LIBS = libexpr libmain libstore libutil libformat + +$(eval $(call install-symlink, nix, $(bindir)/nix-hash)) diff --git a/src/nix/main.cc b/src/nix/main.cc new file mode 100644 index 000000000000..20d3ea5c2555 --- /dev/null +++ b/src/nix/main.cc @@ -0,0 +1,55 @@ +#include + +#include "command.hh" +#include "common-args.hh" +#include "eval.hh" +#include "globals.hh" +#include "legacy.hh" +#include "shared.hh" +#include "store-api.hh" + +namespace nix { + +struct NixArgs : virtual MultiCommand, virtual MixCommonArgs +{ + NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") + { + mkFlag('h', "help", "show usage information", [=]() { + printHelp(programName, std::cout); + throw Exit(); + }); + + mkFlag(0, "version", "show version information", std::bind(printVersion, programName)); + } +}; + +void mainWrapped(int argc, char * * argv) +{ + initNix(); + initGC(); + + string programName = baseNameOf(argv[0]); + + { + auto legacy = (*RegisterLegacyCommand::commands)[programName]; + if (legacy) return legacy(argc, argv); + } + + NixArgs args; + + args.parseCmdline(argvToStrings(argc, argv)); + + assert(args.command); + + args.command->prepare(); + args.command->run(); +} + +} + +int main(int argc, char * * argv) +{ + return nix::handleExceptions(argv[0], [&]() { + nix::mainWrapped(argc, argv); + }); +} -- cgit 1.4.1 From 206bbb5dc9f714db51cdae973de9efef71d96cf7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 9 Feb 2016 21:34:24 +0100 Subject: Add basic "nix build" command Currently only builds by attribute from or the specified file, e.g. "nix build hello". --- src/nix/build.cc | 46 ++++++++++++++++++++++++++++++ src/nix/installables.cc | 75 +++++++++++++++++++++++++++++++++++++++++++++++++ src/nix/installables.hh | 38 +++++++++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 src/nix/build.cc create mode 100644 src/nix/installables.cc create mode 100644 src/nix/installables.hh diff --git a/src/nix/build.cc b/src/nix/build.cc new file mode 100644 index 000000000000..35603f557a98 --- /dev/null +++ b/src/nix/build.cc @@ -0,0 +1,46 @@ +#include "command.hh" +#include "common-args.hh" +#include "installables.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdBuild : StoreCommand, MixDryRun, MixInstallables +{ + CmdBuild() + { + } + + std::string name() override + { + return "build"; + } + + std::string description() override + { + return "build a derivation or fetch a store path"; + } + + void run(ref store) override + { + auto elems = evalInstallables(store); + + PathSet pathsToBuild; + + for (auto & elem : elems) { + if (elem.isDrv) + pathsToBuild.insert(elem.drvPath); + else + pathsToBuild.insert(elem.outPaths.begin(), elem.outPaths.end()); + } + + printMissing(store, pathsToBuild); + + if (dryRun) return; + + store->buildPaths(pathsToBuild); + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/src/nix/installables.cc b/src/nix/installables.cc new file mode 100644 index 000000000000..fb5a515825aa --- /dev/null +++ b/src/nix/installables.cc @@ -0,0 +1,75 @@ +#include "attr-path.hh" +#include "common-opts.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "installables.hh" +#include "store-api.hh" + +namespace nix { + +UserEnvElems MixInstallables::evalInstallables(ref store) +{ + UserEnvElems res; + + for (auto & installable : installables) { + + if (std::string(installable, 0, 1) == "/") { + + if (isStorePath(installable)) { + + if (isDerivation(installable)) { + UserEnvElem elem; + // FIXME: handle empty case, drop version + elem.attrPath = {storePathToName(installable)}; + elem.isDrv = true; + elem.drvPath = installable; + res.push_back(elem); + } + + else { + UserEnvElem elem; + // FIXME: handle empty case, drop version + elem.attrPath = {storePathToName(installable)}; + elem.isDrv = false; + elem.outPaths = {installable}; + res.push_back(elem); + } + } + + else + throw UsageError(format("don't know what to do with ‘%1%’") % installable); + } + + else { + + EvalState state({}, store); + + Expr * e = state.parseExprFromFile(resolveExprPath(lookupFileArg(state, file))); + + Value vRoot; + state.eval(e, vRoot); + + std::map autoArgs_; + Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + + Value & v(*findAlongAttrPath(state, installable, autoArgs, vRoot)); + state.forceValue(v); + + DrvInfos drvs; + getDerivations(state, v, "", autoArgs, drvs, false); + + for (auto & i : drvs) { + UserEnvElem elem; + elem.isDrv = true; + elem.drvPath = i.queryDrvPath(); + res.push_back(elem); + } + } + } + + return res; +} + +} diff --git a/src/nix/installables.hh b/src/nix/installables.hh new file mode 100644 index 000000000000..5eb897d46148 --- /dev/null +++ b/src/nix/installables.hh @@ -0,0 +1,38 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +struct UserEnvElem +{ + Strings attrPath; + + // FIXME: should use boost::variant or so. + bool isDrv; + + // Derivation case: + Path drvPath; + StringSet outputNames; + + // Non-derivation case: + PathSet outPaths; +}; + +typedef std::vector UserEnvElems; + +struct MixInstallables : virtual Args +{ + Strings installables; + Path file = ""; + + MixInstallables() + { + mkFlag('f', "file", "file", "evaluate FILE rather than the default", &file); + expectArgs("installables", &installables); + } + + UserEnvElems evalInstallables(ref store); +}; + +} -- cgit 1.4.1 From 9b05d5848c2fce73b75b3411e362c2bd48d53dcb Mon Sep 17 00:00:00 2001 From: Tristan Hume Date: Sat, 13 Feb 2016 23:24:55 -0500 Subject: Fix to-base16 description --- src/nix/hash.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index fff945bd33d9..452aa4ef67c0 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -69,7 +69,7 @@ struct CmdToBase : Command { return toBase32 ? "convert a hash to base-32 representation" - : "convert a hash to base-32 representation"; + : "convert a hash to base-16 representation"; } void run() override -- cgit 1.4.1 From 7873cfb18df352d75628ea8f6f426a0c8e066ab7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 25 Feb 2016 11:55:05 +0100 Subject: Fix build --- src/nix/build.cc | 2 +- src/nix/command.cc | 2 +- src/nix/hash.cc | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 35603f557a98..812464d7582b 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -43,4 +43,4 @@ struct CmdBuild : StoreCommand, MixDryRun, MixInstallables } }; -static RegisterCommand r1(make_ref()); +static RegisterCommand r1(make_ref()); diff --git a/src/nix/command.cc b/src/nix/command.cc index 698863349334..9c80f43093c5 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -59,7 +59,7 @@ bool MultiCommand::processArgs(const Strings & args, bool finish) void StoreCommand::run() { - run(openStore(reserveSpace)); + run(openStore()); } } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 452aa4ef67c0..5dd891e8add3 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -45,8 +45,8 @@ struct CmdHash : Command } }; -static RegisterCommand r1(make_ref(CmdHash::mFile)); -static RegisterCommand r2(make_ref(CmdHash::mPath)); +static RegisterCommand r1(make_ref(CmdHash::mFile)); +static RegisterCommand r2(make_ref(CmdHash::mPath)); struct CmdToBase : Command { @@ -82,8 +82,8 @@ struct CmdToBase : Command } }; -static RegisterCommand r3(make_ref(false)); -static RegisterCommand r4(make_ref(true)); +static RegisterCommand r3(make_ref(false)); +static RegisterCommand r4(make_ref(true)); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) -- cgit 1.4.1 From c5bc57186192277b42746bbfba7bb799d9246da3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 25 Feb 2016 13:31:34 +0100 Subject: Fix short boolean flags --- src/libutil/args.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 0c7729d0b6b1..4469a046d28a 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -86,7 +86,7 @@ public: void mkFlag(char shortName, const std::string & name, const std::string & description, bool * dest) { - mkFlag(0, name, description, dest, true); + mkFlag(shortName, name, description, dest, true); } void mkFlag(char shortName, const std::string & longName, -- cgit 1.4.1 From 00b2c05749f9777a91543cad9ecafb0981389be6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 25 Feb 2016 17:57:00 +0100 Subject: nix: Add commands to query contents of NARs / binary caches For example, $ NIX_REMOTE=file:///my-cache nix ls-store -lR /nix/store/f4kbgl8shhyy76rkk3nbxr0lz8d2ip7q-binutils-2.23.1 dr-xr-xr-x 0 ./bin -r-xr-xr-x 30748 ./bin/addr2line -r-xr-xr-x 66973 ./bin/ar ... Similarly, "nix ls-nar" lists the contents of a NAR file, "nix cat-nar" extracts a file from a NAR file, and "nix cat-store" extract a file from a Nix store. --- src/nix/cat.cc | 74 ++++++++++++++++++++++++++++++++++ src/nix/ls.cc | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 src/nix/cat.cc create mode 100644 src/nix/ls.cc diff --git a/src/nix/cat.cc b/src/nix/cat.cc new file mode 100644 index 000000000000..2405a8cb44ef --- /dev/null +++ b/src/nix/cat.cc @@ -0,0 +1,74 @@ +#include "command.hh" +#include "store-api.hh" +#include "fs-accessor.hh" +#include "nar-accessor.hh" + +using namespace nix; + +struct MixCat : virtual Args +{ + std::string path; + + void cat(ref accessor) + { + auto st = accessor->stat(path); + if (st.type == FSAccessor::Type::tMissing) + throw Error(format("path ‘%1%’ does not exist") % path); + if (st.type != FSAccessor::Type::tRegular) + throw Error(format("path ‘%1%’ is not a regular file") % path); + + std::cout << accessor->readFile(path); + } +}; + +struct CmdCatStore : StoreCommand, MixCat +{ + CmdCatStore() + { + expectArg("path", &path); + } + + std::string name() override + { + return "cat-store"; + } + + std::string description() override + { + return "print the contents of a store file on stdout"; + } + + void run(ref store) override + { + cat(store->getFSAccessor()); + } +}; + +struct CmdCatNar : StoreCommand, MixCat +{ + Path narPath; + + CmdCatNar() + { + expectArg("nar", &narPath); + expectArg("path", &path); + } + + std::string name() override + { + return "cat-nar"; + } + + std::string description() override + { + return "print the contents of a file inside a NAR file"; + } + + void run(ref store) override + { + cat(makeNarAccessor(make_ref(readFile(narPath)))); + } +}; + +static RegisterCommand r1(make_ref()); +static RegisterCommand r2(make_ref()); diff --git a/src/nix/ls.cc b/src/nix/ls.cc new file mode 100644 index 000000000000..3476dfb05287 --- /dev/null +++ b/src/nix/ls.cc @@ -0,0 +1,123 @@ +#include "command.hh" +#include "store-api.hh" +#include "fs-accessor.hh" +#include "nar-accessor.hh" + +using namespace nix; + +struct MixLs : virtual Args +{ + std::string path; + + bool recursive = false; + bool verbose = false; + bool showDirectory = false; + + MixLs() + { + mkFlag('R', "recursive", "list subdirectories recursively", &recursive); + mkFlag('l', "long", "show more file information", &verbose); + mkFlag('d', "directory", "show directories rather than their contents", &showDirectory); + } + + void list(ref accessor) + { + std::function doPath; + + auto showFile = [&](const Path & curPath, const std::string & relPath) { + if (verbose) { + auto st = accessor->stat(curPath); + std::string tp = + st.type == FSAccessor::Type::tRegular ? + (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : + st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : + "dr-xr-xr-x"; + std::cout << + (format("%s %20d %s") % tp % st.fileSize % relPath); + if (st.type == FSAccessor::Type::tSymlink) + std::cout << " -> " << accessor->readLink(curPath) + ; + std::cout << "\n"; + if (recursive && st.type == FSAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); + } else { + std::cout << relPath << "\n"; + if (recursive) { + auto st = accessor->stat(curPath); + if (st.type == FSAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); + } + } + }; + + doPath = [&](const FSAccessor::Stat & st , const Path & curPath, + const std::string & relPath, bool showDirectory) + { + if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { + auto names = accessor->readDirectory(curPath); + for (auto & name : names) + showFile(curPath + "/" + name, relPath + "/" + name); + } else + showFile(curPath, relPath); + }; + + auto st = accessor->stat(path); + if (st.type == FSAccessor::Type::tMissing) + throw Error(format("path ‘%1%’ does not exist") % path); + doPath(st, path, + st.type == FSAccessor::Type::tDirectory ? "." : baseNameOf(path), + showDirectory); + } +}; + +struct CmdLsStore : StoreCommand, MixLs +{ + CmdLsStore() + { + expectArg("path", &path); + } + + std::string name() override + { + return "ls-store"; + } + + std::string description() override + { + return "show information about a store path"; + } + + void run(ref store) override + { + list(store->getFSAccessor()); + } +}; + +struct CmdLsNar : Command, MixLs +{ + Path narPath; + + CmdLsNar() + { + expectArg("nar", &narPath); + expectArg("path", &path); + } + + std::string name() override + { + return "ls-nar"; + } + + std::string description() override + { + return "show information about the contents of a NAR file"; + } + + void run() override + { + list(makeNarAccessor(make_ref(readFile(narPath)))); + } +}; + +static RegisterCommand r1(make_ref()); +static RegisterCommand r2(make_ref()); -- cgit 1.4.1 From cca4a8dc1a622ab086639e5c09347303c062922e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2016 15:20:10 +0100 Subject: importPaths(): Optionally add NARs to binary cache accessor This enables an optimisation in hydra-queue-runner, preventing a download of a NAR it just uploaded to the cache when reading files like hydra-build-products. --- perl/lib/Nix/Store.xs | 2 +- src/libstore/binary-cache-store.cc | 66 +++++++++++++++++++++----------------- src/libstore/binary-cache-store.hh | 5 +-- src/libstore/local-store.cc | 3 +- src/libstore/local-store.hh | 3 +- src/libstore/remote-store.cc | 3 +- src/libstore/remote-store.hh | 3 +- src/libstore/store-api.hh | 7 ++-- src/nix-daemon/nix-daemon.cc | 6 ++-- src/nix-store/nix-store.cc | 4 +-- 10 files changed, 58 insertions(+), 44 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 44c88a87bfc6..bb322875d0ca 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -186,7 +186,7 @@ void importPaths(int fd) PPCODE: try { FdSource source(fd); - store()->importPaths(false, source); + store()->importPaths(false, source, 0); } catch (Error & e) { croak("%s", e.what()); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 6a4e3a560578..02e73d2ce1fa 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -183,7 +183,8 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink sink << exportMagic << storePath << res.references << res.deriver << 0; } -Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source) +Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) { assert(!requireSignature); Paths res; @@ -191,7 +192,7 @@ Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source) unsigned long long n = readLongLong(source); if (n == 0) break; if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’"); - res.push_back(importPath(source)); + res.push_back(importPath(source, accessor)); } return res; } @@ -215,34 +216,6 @@ struct NopSink : ParseSink { }; -Path BinaryCacheStore::importPath(Source & source) -{ - /* FIXME: some cut&paste of LocalStore::importPath(). */ - - /* Extract the NAR from the source. */ - TeeSource tee(source); - NopSink sink; - parseDump(sink, tee); - - uint32_t magic = readInt(source); - if (magic != exportMagic) - throw Error("Nix archive cannot be imported; wrong format"); - - ValidPathInfo info; - info.path = readStorePath(source); - - info.references = readStorePaths(source); - - readString(source); // deriver, don't care - - bool haveSignature = readInt(source) == 1; - assert(!haveSignature); - - addToCache(info, tee.data); - - return info.path; -} - ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath) { return ValidPathInfo(readNarInfo(storePath)); @@ -416,4 +389,37 @@ ref BinaryCacheStore::getFSAccessor() std::dynamic_pointer_cast(shared_from_this()))); } +Path BinaryCacheStore::importPath(Source & source, std::shared_ptr accessor) +{ + /* FIXME: some cut&paste of LocalStore::importPath(). */ + + /* Extract the NAR from the source. */ + TeeSource tee(source); + NopSink sink; + parseDump(sink, tee); + + uint32_t magic = readInt(source); + if (magic != exportMagic) + throw Error("Nix archive cannot be imported; wrong format"); + + ValidPathInfo info; + info.path = readStorePath(source); + + info.references = readStorePaths(source); + + readString(source); // deriver, don't care + + bool haveSignature = readInt(source) == 1; + assert(!haveSignature); + + addToCache(info, tee.data); + + auto accessor_ = std::dynamic_pointer_cast(accessor); + if (accessor_) + // FIXME: more gratuitous string copying + accessor_->nars.emplace(info.path, makeNarAccessor(make_ref(tee.data))); + + return info.path; +} + } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index c6f319d1bc82..6feb84cd2b10 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -127,9 +127,10 @@ public: void exportPath(const Path & path, bool sign, Sink & sink) override; - Paths importPaths(bool requireSignature, Source & source) override; + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) override; - Path importPath(Source & source); + Path importPath(Source & source, std::shared_ptr accessor); void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 33f2569128aa..9a570668128e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1689,7 +1689,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) } -Paths LocalStore::importPaths(bool requireSignature, Source & source) +Paths LocalStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) { Paths res; while (true) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fded63ab38e1..c7ea9e5038b4 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -147,7 +147,8 @@ public: void exportPath(const Path & path, bool sign, Sink & sink) override; - Paths importPaths(bool requireSignature, Source & source) override; + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a1ac169c2b9e..82b7cfd7c17d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -406,7 +406,8 @@ void RemoteStore::exportPath(const Path & path, bool sign, } -Paths RemoteStore::importPaths(bool requireSignature, Source & source) +Paths RemoteStore::importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) { auto conn(connections->get()); conn->to << wopImportPaths; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 0019cd8f9f94..85c8292c7698 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -65,7 +65,8 @@ public: void exportPath(const Path & path, bool sign, Sink & sink) override; - Paths importPaths(bool requireSignature, Source & source) override; + Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 347387f03403..488f32e16456 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -227,8 +227,11 @@ public: void exportPaths(const Paths & paths, bool sign, Sink & sink); /* Import a sequence of NAR dumps created by exportPaths() into - the Nix store. */ - virtual Paths importPaths(bool requireSignature, Source & source) = 0; + the Nix store. Optionally, the contents of the NARs are + preloaded into the specified FS accessor to speed up subsequent + access. */ + virtual Paths importPaths(bool requireSignature, Source & source, + std::shared_ptr accessor) = 0; /* For each path, if it's a derivation, build it. Building a derivation means ensuring that the output paths are valid. If diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index e3c12ea0e0ea..5189c9b4c179 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -310,7 +310,7 @@ static void performOp(ref store, bool trusted, unsigned int clientVe case wopImportPaths: { startWork(); TunnelSource source(from); - Paths paths = store->importPaths(!trusted, source); + Paths paths = store->importPaths(!trusted, source, 0); stopWork(); to << paths; break; @@ -322,8 +322,8 @@ static void performOp(ref store, bool trusted, unsigned int clientVe if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode)readInt(from); - /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + /* Repairing is not atomic, so disallowed for "untrusted" + clients. */ if (mode == bmRepair && !trusted) throw Error("repairing is not supported when building through the Nix daemon"); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 48c2865e5ecc..a258c77c326f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -732,7 +732,7 @@ static void opImport(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - Paths paths = store->importPaths(requireSignature, source); + Paths paths = store->importPaths(requireSignature, source, 0); for (auto & i : paths) cout << format("%1%\n") % i << std::flush; @@ -935,7 +935,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); - store->importPaths(false, in); + store->importPaths(false, in, 0); out << 1; // indicate success break; } -- cgit 1.4.1 From 89a2fa68aca83082b50132dd98a170bcce82dae9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2016 16:16:08 +0100 Subject: FdSource: track number of bytes read --- src/libutil/serialise.cc | 3 ++- src/libutil/serialise.hh | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index c9620e2bf32a..a3cd4ff0d02c 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -64,9 +64,9 @@ static void warnLargeDump() void FdSink::write(const unsigned char * data, size_t len) { + written += len; static bool warned = false; if (warn && !warned) { - written += len; if (written > threshold) { warnLargeDump(); warned = true; @@ -131,6 +131,7 @@ size_t FdSource::readUnbuffered(unsigned char * data, size_t len) } while (n == -1 && errno == EINTR); if (n == -1) { _good = false; throw SysError("reading from file"); } if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } + read += n; return n; } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 9e269f3923ea..0fdc4037b2d7 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -76,11 +76,11 @@ struct BufferedSource : Source struct FdSink : BufferedSink { int fd; - bool warn; - size_t written; + bool warn = false; + size_t written = 0; - FdSink() : fd(-1), warn(false), written(0) { } - FdSink(int fd) : fd(fd), warn(false), written(0) { } + FdSink() : fd(-1) { } + FdSink(int fd) : fd(fd) { } ~FdSink(); void write(const unsigned char * data, size_t len) override; @@ -96,6 +96,8 @@ private: struct FdSource : BufferedSource { int fd; + size_t read = 0; + FdSource() : fd(-1) { } FdSource(int fd) : fd(fd) { } size_t readUnbuffered(unsigned char * data, size_t len) override; -- cgit 1.4.1 From 0a62d9b3d73afb8a5111cdecb23a0a4d6954493d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2016 21:43:59 +0100 Subject: Remove bad assertion --- src/libstore/nar-accessor.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 88b027ff3b7d..ff7890af8c80 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -51,7 +51,6 @@ struct NarIndexer : ParseSink, StringSource void preallocateContents(unsigned long long size) override { - assert(currentPath != ""); currentStart = string(s, pos, 16); members.emplace(currentPath, NarMember{FSAccessor::Type::tRegular, isExec, pos, size}); -- cgit 1.4.1 From 6055d84bebe6eecc061ef295c1fd21ceb77e7674 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 28 Feb 2016 03:26:46 +0100 Subject: Fix reading symlinks The st_size field of a symlink doesn't have to be correct, e.g. for /dev/fd symlinks. --- src/libutil/util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 3becbbabc2bf..25246a3e89a9 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -209,7 +209,7 @@ Path readLink(const Path & path) else if (rlsize > st.st_size) throw Error(format("symbolic link ‘%1%’ size overflow %2% > %3%") % path % rlsize % st.st_size); - return string(buf, st.st_size); + return string(buf, rlsize); } -- cgit 1.4.1 From 68a541498258949231e35544501e11c6003f2dce Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Feb 2016 16:11:11 +0100 Subject: Make store implementations pluggable This for instance allows hydra-queue-runner to add the S3 backend at runtime. --- src/libstore/local-binary-cache-store.cc | 9 +++++++ src/libstore/store-api.cc | 43 ++++++++++++++++++-------------- src/libstore/store-api.hh | 17 +++++++++++++ 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 5714688e02df..9fae80870ade 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -41,4 +41,13 @@ std::string LocalBinaryCacheStore::getFile(const std::string & path) return readFile(binaryCacheDir + "/" + path); } +static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr { + if (std::string(uri, 0, 7) != "file://") return 0; + auto store = std::make_shared(std::shared_ptr(0), + "", "", // FIXME: allow the signing key to be set + std::string(uri, 7)); + store->init(); + return store; +}); + } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 24c05b8b4873..378233654225 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -314,27 +314,38 @@ void Store::exportPaths(const Paths & paths, #include "local-store.hh" #include "remote-store.hh" -#include "local-binary-cache-store.hh" namespace nix { +RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; + + ref openStoreAt(const std::string & uri) { - if (std::string(uri, 0, 7) == "file://") { - auto store = make_ref(std::shared_ptr(0), - "", "", // FIXME: allow the signing key to be set - std::string(uri, 7)); - store->init(); - return store; + for (auto fun : *RegisterStoreImplementation::implementations) { + auto store = fun(uri); + if (store) return ref(store); } + throw Error(format("don't know how to open Nix store ‘%s’") % uri); +} + + +ref openStore() +{ + return openStoreAt(getEnv("NIX_REMOTE")); +} + + +static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr { enum { mDaemon, mLocal, mAuto } mode; - mode = - uri == "daemon" ? mDaemon : - uri == "local" ? mLocal : mAuto; + if (uri == "daemon") mode = mDaemon; + else if (uri == "local") mode = mLocal; + else if (uri == "") mode = mAuto; + else return 0; if (mode == mAuto) { if (LocalStore::haveWriteAccess()) @@ -346,15 +357,9 @@ ref openStoreAt(const std::string & uri) } return mode == mDaemon - ? (ref) make_ref() - : (ref) make_ref(); -} - - -ref openStore() -{ - return openStoreAt(getEnv("NIX_REMOTE")); -} + ? std::shared_ptr(std::make_shared()) + : std::shared_ptr(std::make_shared()); +}); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 488f32e16456..97e834ed2bd8 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -453,6 +453,23 @@ ref openStoreAt(const std::string & uri); ref openStore(); +/* Store implementation registration. */ +typedef std::function(const std::string & uri)> OpenStore; + +struct RegisterStoreImplementation +{ + typedef std::vector Implementations; + static Implementations * implementations; + + RegisterStoreImplementation(OpenStore fun) + { + if (!implementations) implementations = new Implementations; + implementations->push_back(fun); + } +}; + + + /* Display a set of paths in human-readable form (i.e., between quotes and separated by commas). */ string showPaths(const PathSet & paths); -- cgit 1.4.1 From 0402b6398d879889a621d60d049f8d81ac4ed86a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Feb 2016 16:14:39 +0100 Subject: Eliminate local-binary-cache-store.hh --- src/libstore/local-binary-cache-store.cc | 26 +++++++++++++++++++++++++- src/libstore/local-binary-cache-store.hh | 31 ------------------------------- 2 files changed, 25 insertions(+), 32 deletions(-) delete mode 100644 src/libstore/local-binary-cache-store.hh diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 9fae80870ade..a10c9d1069d5 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -1,7 +1,31 @@ -#include "local-binary-cache-store.hh" +#include "binary-cache-store.hh" namespace nix { +class LocalBinaryCacheStore : public BinaryCacheStore +{ +private: + + Path binaryCacheDir; + +public: + + LocalBinaryCacheStore(std::shared_ptr localStore, + const Path & secretKeyFile, const Path & publicKeyFile, + const Path & binaryCacheDir); + + void init() override; + +protected: + + bool fileExists(const std::string & path) override; + + void upsertFile(const std::string & path, const std::string & data) override; + + std::string getFile(const std::string & path) override; + +}; + LocalBinaryCacheStore::LocalBinaryCacheStore(std::shared_ptr localStore, const Path & secretKeyFile, const Path & publicKeyFile, const Path & binaryCacheDir) diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh deleted file mode 100644 index 0303ebe734a6..000000000000 --- a/src/libstore/local-binary-cache-store.hh +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include "binary-cache-store.hh" - -namespace nix { - -class LocalBinaryCacheStore : public BinaryCacheStore -{ -private: - - Path binaryCacheDir; - -public: - - LocalBinaryCacheStore(std::shared_ptr localStore, - const Path & secretKeyFile, const Path & publicKeyFile, - const Path & binaryCacheDir); - - void init() override; - -protected: - - bool fileExists(const std::string & path) override; - - void upsertFile(const std::string & path, const std::string & data) override; - - std::string getFile(const std::string & path) override; - -}; - -} -- cgit 1.4.1 From 201b48de60751979835037a4b4f78128ba3fb7b3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Feb 2016 18:15:20 +0100 Subject: Add an HTTP binary cache store Allowing stuff like NIX_REMOTE=https://cache.nixos.org nix-store -qR /nix/store/x1p1gl3a4kkz5ci0nfbayjqlqmczp1kq-geeqie-1.1 or NIX_REMOTE=https://cache.nixos.org nix-store --export /nix/store/x1p1gl3a4kkz5ci0nfbayjqlqmczp1kq-geeqie-1.1 | nix-store --import --- src/libexpr/common-opts.cc | 2 +- src/libexpr/parser.y | 2 +- src/libexpr/primops.cc | 2 +- src/libstore/builtins.cc | 4 +- src/libstore/download.cc | 124 +++++++++++++++++++------------ src/libstore/download.hh | 23 +++++- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- 7 files changed, 100 insertions(+), 59 deletions(-) diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc index 68ab4b5cdcbf..8a7989aac663 100644 --- a/src/libexpr/common-opts.cc +++ b/src/libexpr/common-opts.cc @@ -55,7 +55,7 @@ bool parseSearchPathArg(Strings::iterator & i, Path lookupFileArg(EvalState & state, string s) { if (isUri(s)) - return downloadFileCached(state.store, s, true); + return makeDownloader()->downloadCached(state.store, s, true); else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 80ecd44c5981..11dc7bb5ccdf 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -613,7 +613,7 @@ void EvalState::addToSearchPath(const string & s, bool warn) } if (isUri(path)) - path = downloadFileCached(store, path, true); + path = makeDownloader()->downloadCached(store, path, true); path = absPath(path); if (pathExists(path)) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3c899d769253..5bfb95be6849 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1703,7 +1703,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, } else url = state.forceStringNoCtx(*args[0], pos); - Path res = downloadFileCached(state.store, url, unpack); + Path res = makeDownloader()->downloadCached(state.store, url, unpack); mkString(v, res, PathSet({res})); } diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc index a1c4b48bf62e..c22c44f3c7e3 100644 --- a/src/libstore/builtins.cc +++ b/src/libstore/builtins.cc @@ -17,9 +17,9 @@ void builtinFetchurl(const BasicDerivation & drv) options.verifyTLS = false; /* Show a progress indicator, even though stderr is not a tty. */ - options.forceProgress = true; + options.showProgress = DownloadOptions::yes; - auto data = downloadFile(url->second, options); + auto data = makeDownloader()->download(url->second, options); auto out = drv.env.find("out"); if (out == drv.env.end()) throw Error("attribute ‘url’ missing"); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index e754e82fb27f..4776d0091685 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -18,7 +18,7 @@ double getTime() return tv.tv_sec + (tv.tv_usec / 1000000.0); } -struct Curl +struct CurlDownloader : public Downloader { CURL * curl; string data; @@ -30,37 +30,40 @@ struct Curl double prevProgressTime{0}, startTime{0}; unsigned int moveBack{1}; - static size_t writeCallback(void * contents, size_t size, size_t nmemb, void * userp) + size_t writeCallback(void * contents, size_t size, size_t nmemb) { - Curl & c(* (Curl *) userp); size_t realSize = size * nmemb; - c.data.append((char *) contents, realSize); + data.append((char *) contents, realSize); return realSize; } - static size_t headerCallback(void * contents, size_t size, size_t nmemb, void * userp) + static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) + { + return ((CurlDownloader *) userp)->writeCallback(contents, size, nmemb); + } + + size_t headerCallback(void * contents, size_t size, size_t nmemb) { - Curl & c(* (Curl *) userp); size_t realSize = size * nmemb; string line = string((char *) contents, realSize); printMsg(lvlVomit, format("got header: %1%") % trim(line)); if (line.compare(0, 5, "HTTP/") == 0) { // new response starts - c.etag = ""; + etag = ""; auto ss = tokenizeString>(line, " "); - c.status = ss.size() >= 2 ? ss[1] : ""; + status = ss.size() >= 2 ? ss[1] : ""; } else { auto i = line.find(':'); if (i != string::npos) { string name = trim(string(line, 0, i)); if (name == "ETag") { // FIXME: case - c.etag = trim(string(line, i + 1)); + etag = trim(string(line, i + 1)); /* Hack to work around a GitHub bug: it sends ETags, but ignores If-None-Match. So if we get the expected ETag on a 200 response, then shut down the connection because we already have the data. */ - printMsg(lvlDebug, format("got ETag: %1%") % c.etag); - if (c.etag == c.expectedETag && c.status == "200") { + printMsg(lvlDebug, format("got ETag: %1%") % etag); + if (etag == expectedETag && status == "200") { printMsg(lvlDebug, format("shutting down on 200 HTTP response with expected ETag")); return 0; } @@ -70,6 +73,11 @@ struct Curl return realSize; } + static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) + { + return ((CurlDownloader *) userp)->headerCallback(contents, size, nmemb); + } + int progressCallback(double dltotal, double dlnow) { if (showProgress) { @@ -88,45 +96,48 @@ struct Curl return _isInterrupted; } - static int progressCallback_(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) + static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) { - Curl & c(* (Curl *) userp); - return c.progressCallback(dltotal, dlnow); + return ((CurlDownloader *) userp)->progressCallback(dltotal, dlnow); } - Curl() + CurlDownloader() { requestHeaders = 0; curl = curl_easy_init(); - if (!curl) throw Error("unable to initialize curl"); + if (!curl) throw nix::Error("unable to initialize curl"); + } + + ~CurlDownloader() + { + if (curl) curl_easy_cleanup(curl); + if (requestHeaders) curl_slist_free_all(requestHeaders); + } + + bool fetch(const string & url, const DownloadOptions & options) + { + showProgress = + options.showProgress == DownloadOptions::yes || + (options.showProgress == DownloadOptions::automatic && isatty(STDERR_FILENO)); + + curl_easy_reset(curl); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str()); curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &curl); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallbackWrapper); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) this); - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallback); - curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) &curl); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallbackWrapper); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) this); - curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallback_); - curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, (void *) &curl); + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, (void *) this); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - } - - ~Curl() - { - if (curl) curl_easy_cleanup(curl); - if (requestHeaders) curl_slist_free_all(requestHeaders); - } - - bool fetch(const string & url, const DownloadOptions & options) - { - showProgress = options.forceProgress || isatty(STDERR_FILENO); curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); @@ -151,6 +162,9 @@ struct Curl curl_easy_setopt(curl, CURLOPT_HTTPHEADER, requestHeaders); + if (options.head) + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + if (showProgress) { std::cerr << (format("downloading ‘%1%’... ") % url); std::cerr.flush(); @@ -163,34 +177,46 @@ struct Curl std::cerr << "\n"; checkInterrupt(); if (res == CURLE_WRITE_ERROR && etag == options.expectedETag) return false; - if (res != CURLE_OK) - throw DownloadError(format("unable to download ‘%1%’: %2% (%3%)") + + long httpStatus = -1; + if (res == CURLE_HTTP_RETURNED_ERROR) + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); + + if (res != CURLE_OK) { + long httpStatus = 0; + Error err = + httpStatus == 404 ? NotFound : + httpStatus == 403 ? Forbidden : Misc; + throw DownloadError(err, format("unable to download ‘%1%’: %2% (%3%)") % url % curl_easy_strerror(res) % res); + } - long httpStatus = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); if (httpStatus == 304) return false; return true; } + + DownloadResult download(string url, const DownloadOptions & options) override + { + DownloadResult res; + if (fetch(url, options)) { + res.cached = false; + res.data = data; + } else + res.cached = true; + res.etag = etag; + return res; + } }; -DownloadResult downloadFile(string url, const DownloadOptions & options) +ref makeDownloader() { - DownloadResult res; - Curl curl; - if (curl.fetch(url, options)) { - res.cached = false; - res.data = curl.data; - } else - res.cached = true; - res.etag = curl.etag; - return res; + return make_ref(); } -Path downloadFileCached(ref store, const string & url, bool unpack) +Path Downloader::downloadCached(ref store, const string & url, bool unpack) { Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs"; createDirs(cacheDir); @@ -234,7 +260,7 @@ Path downloadFileCached(ref store, const string & url, bool unpack) try { DownloadOptions options; options.expectedETag = expectedETag; - auto res = downloadFile(url, options); + auto res = download(url, options); if (!res.cached) storePath = store->addTextToStore(name, res.data, PathSet(), false); diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 7aec8de73e48..5dd2d2c82dec 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -10,7 +10,8 @@ struct DownloadOptions { string expectedETag; bool verifyTLS{true}; - bool forceProgress{false}; + enum { yes, no, automatic } showProgress{yes}; + bool head{false}; }; struct DownloadResult @@ -21,11 +22,25 @@ struct DownloadResult class Store; -DownloadResult downloadFile(string url, const DownloadOptions & options); +struct Downloader +{ + virtual DownloadResult download(string url, const DownloadOptions & options) = 0; + + Path downloadCached(ref store, const string & url, bool unpack); + + enum Error { NotFound, Forbidden, Misc }; +}; -Path downloadFileCached(ref store, const string & url, bool unpack); +ref makeDownloader(); -MakeError(DownloadError, Error) +class DownloadError : public Error +{ +public: + Downloader::Error error; + DownloadError(Downloader::Error error, const FormatOrString & fs) + : Error(fs), error(error) + { } +}; bool isUri(const string & s); diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index c0c05a60bd78..c65961a15720 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -158,7 +158,7 @@ int main(int argc, char * * argv) auto actualUri = resolveMirrorUri(state, uri); /* Download the file. */ - auto result = downloadFile(actualUri, DownloadOptions()); + auto result = makeDownloader()->download(actualUri, DownloadOptions()); AutoDelete tmpDir(createTempDir(), true); Path tmpFile = (Path) tmpDir + "/tmp"; -- cgit 1.4.1 From 25eea97873396ac1aead2dddcf8050e315c31f80 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Mar 2016 15:45:56 +0100 Subject: Force stack trace for boost format errors --- src/boost/format/exceptions.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/boost/format/exceptions.hpp b/src/boost/format/exceptions.hpp index 79e452449ef8..a7641458c95e 100644 --- a/src/boost/format/exceptions.hpp +++ b/src/boost/format/exceptions.hpp @@ -33,7 +33,7 @@ namespace io { class format_error : public std::exception { public: - format_error() {} + format_error() { abort(); } virtual const char *what() const throw() { return "boost::format_error: " @@ -44,7 +44,7 @@ public: class bad_format_string : public format_error { public: - bad_format_string() {} + bad_format_string() { abort(); } virtual const char *what() const throw() { return "boost::bad_format_string: " @@ -55,7 +55,7 @@ public: class too_few_args : public format_error { public: - too_few_args() {} + too_few_args() { abort(); } virtual const char *what() const throw() { return "boost::too_few_args: " @@ -66,7 +66,7 @@ public: class too_many_args : public format_error { public: - too_many_args() {} + too_many_args() { abort(); } virtual const char *what() const throw() { return "boost::too_many_args: " @@ -78,7 +78,7 @@ public: class out_of_range : public format_error { public: - out_of_range() {} + out_of_range() { abort(); } virtual const char *what() const throw() { return "boost::out_of_range: " -- cgit 1.4.1 From 5a8455c85e35706a34892bea5b2a0c72f25663a1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Mar 2016 18:21:48 +0100 Subject: Provide function required by Hydra --- src/libstore/local-binary-cache-store.cc | 14 +++++++++++--- src/libstore/store-api.hh | 5 +++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index a10c9d1069d5..8590aea185d4 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -65,13 +65,21 @@ std::string LocalBinaryCacheStore::getFile(const std::string & path) return readFile(binaryCacheDir + "/" + path); } +ref openLocalBinaryCacheStore(std::shared_ptr localStore, + const Path & secretKeyFile, const Path & publicKeyFile, + const Path & binaryCacheDir) +{ + auto store = std::make_shared( + localStore, secretKeyFile, publicKeyFile, binaryCacheDir); + store->init(); + return ref(std::shared_ptr(store)); +} + static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr { if (std::string(uri, 0, 7) != "file://") return 0; - auto store = std::make_shared(std::shared_ptr(0), + return openLocalBinaryCacheStore(std::shared_ptr(0), "", "", // FIXME: allow the signing key to be set std::string(uri, 7)); - store->init(); - return store; }); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 97e834ed2bd8..9825d45db102 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -453,6 +453,11 @@ ref openStoreAt(const std::string & uri); ref openStore(); +ref openLocalBinaryCacheStore(std::shared_ptr localStore, + const Path & secretKeyFile, const Path & publicKeyFile, + const Path & binaryCacheDir); + + /* Store implementation registration. */ typedef std::function(const std::string & uri)> OpenStore; -- cgit 1.4.1 From 76f1ba4f3bd8b503a6395649719a444028ff0258 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Mar 2016 18:03:34 +0100 Subject: Add file missing from 201b48de60751979835037a4b4f78128ba3fb7b3 --- release.nix | 2 +- src/libstore/http-binary-cache-store.cc | 76 +++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 src/libstore/http-binary-cache-store.cc diff --git a/release.nix b/release.nix index 79df103ccfc6..c5bc6f57a740 100644 --- a/release.nix +++ b/release.nix @@ -105,7 +105,7 @@ let installFlags = "sysconfdir=$(out)/etc"; - doInstallCheck = true; + doInstallCheck = false; installCheckFlags = "sysconfdir=$(out)/etc"; }); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc new file mode 100644 index 000000000000..78f4497e7665 --- /dev/null +++ b/src/libstore/http-binary-cache-store.cc @@ -0,0 +1,76 @@ +#include "binary-cache-store.hh" +#include "download.hh" + +namespace nix { + +class HttpBinaryCacheStore : public BinaryCacheStore +{ +private: + + Path cacheUri; + + ref downloader; + +public: + + HttpBinaryCacheStore(std::shared_ptr localStore, + const Path & secretKeyFile, const Path & publicKeyFile, + const Path & _cacheUri) + : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile) + , cacheUri(_cacheUri) + , downloader(makeDownloader()) + { + if (cacheUri.back() == '/') + cacheUri.pop_back(); + } + + void init() override + { + // FIXME: do this lazily? + if (!fileExists("nix-cache-info")) + throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri); + } + +protected: + + bool fileExists(const std::string & path) override + { + try { + DownloadOptions options; + options.showProgress = DownloadOptions::no; + options.head = true; + downloader->download(cacheUri + "/" + path, options); + return true; + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound) + return false; + throw; + } + } + + void upsertFile(const std::string & path, const std::string & data) + { + throw Error("uploading to an HTTP binary cache is not supported"); + } + + std::string getFile(const std::string & path) override + { + DownloadOptions options; + options.showProgress = DownloadOptions::no; + return downloader->download(cacheUri + "/" + path, options).data; + } + +}; + +static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr { + if (std::string(uri, 0, 7) != "http://" && + std::string(uri, 0, 8) != "https://") return 0; + auto store = std::make_shared(std::shared_ptr(0), + "", "", // FIXME: allow the signing key to be set + uri); + store->init(); + return store; +}); + +} + -- cgit 1.4.1 From 1b4b16cc6d9585c7bbeb871edc815137baef8f83 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Mar 2016 15:20:25 +0100 Subject: Doh --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index c5bc6f57a740..79df103ccfc6 100644 --- a/release.nix +++ b/release.nix @@ -105,7 +105,7 @@ let installFlags = "sysconfdir=$(out)/etc"; - doInstallCheck = false; + doInstallCheck = true; installCheckFlags = "sysconfdir=$(out)/etc"; }); -- cgit 1.4.1