From bfa41eb6714a7e7c3956389ee063e898bd1f37ff Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Wed, 10 Aug 2016 10:44:39 -0400 Subject: nix-copy-closure: Implement in C++. Tests fail currently because the database is not given proper hashes in the VM --- src/build-remote/build-remote.cc | 1 - src/build-remote/local.mk | 2 +- src/libstore/ssh-store.cc | 8 +++-- src/libstore/store-api.cc | 24 +++++++++++-- src/libstore/store-api.hh | 2 +- src/nix-copy-closure/local.mk | 7 ++++ src/nix-copy-closure/nix-copy-closure.cc | 60 ++++++++++++++++++++++++++++++++ 7 files changed, 97 insertions(+), 7 deletions(-) create mode 100644 src/nix-copy-closure/local.mk create mode 100755 src/nix-copy-closure/nix-copy-closure.cc (limited to 'src') diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 98ccc3ddc377..1ac9711a1103 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -9,7 +9,6 @@ #include "shared.hh" #include "pathlocks.hh" #include "globals.hh" -#include "serve-protocol.hh" #include "serialise.hh" #include "store-api.hh" #include "derivations.hh" diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk index 05b8cb451435..62d5a010c247 100644 --- a/src/build-remote/local.mk +++ b/src/build-remote/local.mk @@ -8,4 +8,4 @@ build-remote_LIBS = libmain libutil libformat libstore build-remote_SOURCES := $(d)/build-remote.cc -build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" -Isrc/nix-store +build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 3d01594009a0..cce0458c69f2 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -39,6 +39,8 @@ private: string uri; Path key; + + bool compress; }; SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections) @@ -48,6 +50,7 @@ SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections) , socketPath((Path) tmpDir + "/ssh.sock") , uri(std::move(uri)) , key(get(params, "ssh-key", "")) + , compress(get(params, "compress", "") == "true") { /* open a connection and perform the handshake to verify all is well */ connections->get(); @@ -90,11 +93,12 @@ ref SSHStore::getFSAccessor() ref SSHStore::openConnection() { if ((pid_t) sshMaster == -1) { + auto flags = compress ? "-NMCS" : "-NMS"; sshMaster = startProcess([&]() { if (key.empty()) - execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), uri.c_str(), NULL); + execlp("ssh", "ssh", flags, socketPath.c_str(), uri.c_str(), NULL); else - execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL); + execlp("ssh", "ssh", flags, socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL); throw SysError("starting ssh master"); }); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8fdd62771552..c8ca00f00694 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -4,6 +4,7 @@ #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" +#include "derivations.hh" #include @@ -699,8 +700,27 @@ std::list> getDefaultSubstituters() } -void copyPaths(ref from, ref to, const Paths & storePaths) -{ +void copyPaths(ref from, ref to, const Paths & storePaths, bool substitute) +{ + if (substitute) { + /* Filter out .drv files (we don't want to build anything). */ + PathSet paths2; + for (auto & path : storePaths) + if (!isDerivation(path)) paths2.insert(path); + unsigned long long downloadSize, narSize; + PathSet willBuild, willSubstitute, unknown; + to->queryMissing(PathSet(paths2.begin(), paths2.end()), + willBuild, willSubstitute, unknown, downloadSize, narSize); + /* FIXME: should use ensurePath(), but it only + does one path at a time. */ + if (!willSubstitute.empty()) + try { + to->buildPaths(willSubstitute); + } catch (Error & e) { + printMsg(lvlError, format("warning: %1%") % e.msg()); + } + } + std::string copiedLabel = "copied"; logger->setExpected(copiedLabel, storePaths.size()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ec3bf5a6fd83..30ee433bf074 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -625,7 +625,7 @@ void removeTempRoots(); ref openStore(const std::string & uri = getEnv("NIX_REMOTE")); -void copyPaths(ref from, ref to, const Paths & storePaths); +void copyPaths(ref from, ref to, const Paths & storePaths, bool substitute = false); enum StoreType { tDaemon, diff --git a/src/nix-copy-closure/local.mk b/src/nix-copy-closure/local.mk new file mode 100644 index 000000000000..42bb34dd8201 --- /dev/null +++ b/src/nix-copy-closure/local.mk @@ -0,0 +1,7 @@ +programs += nix-copy-closure + +nix-copy-closure_DIR := $(d) + +nix-copy-closure_LIBS = libmain libutil libformat libstore + +nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc new file mode 100755 index 000000000000..b7e997ca4b0c --- /dev/null +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -0,0 +1,60 @@ +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +int main(int argc, char ** argv) +{ + return handleExceptions(argv[0], [&]() { + initNix(); + auto gzip = false; + auto toMode = true; + auto includeOutputs = false; + auto dryRun = false; + auto useSubstitutes = false; + auto sshHost = string{}; + auto storePaths = PathSet{}; + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-copy-closure"); + else if (*arg == "--version") + printVersion("nix-copy-closure"); + else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") { + if (*arg != "--gzip") + printMsg(lvlError, format("Warning: ‘%1%’ is not implemented, falling back to gzip") % *arg); + gzip = true; + } else if (*arg == "--from") + toMode = false; + else if (*arg == "--to") + toMode = true; + else if (*arg == "--include-outputs") + includeOutputs = true; + else if (*arg == "--show-progress") + printMsg(lvlError, "Warning: ‘--show-progress’ is not implemented"); + else if (*arg == "--dry-run") + dryRun = true; + else if (*arg == "--use-substitutes" || *arg == "-s") + useSubstitutes = true; + else if (sshHost.empty()) + sshHost = *arg; + else + storePaths.insert(*arg); + return true; + }); + if (sshHost.empty()) + throw UsageError("no host name specified"); + + auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : ""); + auto to = toMode ? openStore(remoteUri) : openStore(); + auto from = toMode ? openStore() : openStore(remoteUri); + if (includeOutputs) { + auto newPaths = PathSet{}; + for (const auto & p : storePaths) { + auto outputs = from->queryDerivationOutputs(p); + newPaths.insert(outputs.begin(), outputs.end()); + } + storePaths.insert(newPaths.begin(), newPaths.end()); + } + copyPaths(from, to, Paths(storePaths.begin(), storePaths.end()), useSubstitutes); + }); +} -- cgit 1.4.1