nix-copy-closure: Implement in C++.

Tests fail currently because the database is not given proper hashes in the VM

11 files changed, 101 insertions, 116 deletions

diff --git a/.gitignore b/.gitignore
index 92f95fe1fc..a8abc64ada 100644
--- a/.gitignore
+++ b/.gitignore
@@ -81,6 +81,8 @@ Makefile.config
 # /src/nix-build/
 /src/nix-build/nix-build
 
+/src/nix-copy-closure/nix-copy-closure
+
 /src/build-remote/build-remote
 
 # /tests/
diff --git a/Makefile b/Makefile
index 14be271bb1..8390c867d8 100644
--- a/Makefile
+++ b/Makefile
@@ -11,6 +11,7 @@ makefiles = \
   src/nix-env/local.mk \
   src/nix-daemon/local.mk \
   src/nix-collect-garbage/local.mk \
+  src/nix-copy-closure/local.mk \
   src/nix-prefetch-url/local.mk \
   src/buildenv/local.mk \
   src/resolve-system-dependencies/local.mk \
diff --git a/scripts/local.mk b/scripts/local.mk
index ee8ae6845d..9f666bde4d 100644
--- a/scripts/local.mk
+++ b/scripts/local.mk
@@ -1,8 +1,3 @@
-nix_bin_scripts := \
-  $(d)/nix-copy-closure \
-
-bin-scripts += $(nix_bin_scripts)
-
 nix_noinst_scripts := \
   $(d)/build-remote.pl \
   $(d)/nix-http-export.cgi \
@@ -16,4 +11,4 @@ profiledir = $(sysconfdir)/profile.d
 $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644))
 $(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix))
 
-clean-files += $(nix_bin_scripts) $(nix_noinst_scripts)
+clean-files += $(nix_noinst_scripts)
diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in
deleted file mode 100755
index af1d309192..0000000000
--- a/scripts/nix-copy-closure.in
+++ /dev/null
@@ -1,103 +0,0 @@
-#! @perl@ -w @perlFlags@
-
-use utf8;
-use strict;
-use Nix::SSH;
-use Nix::Config;
-use Nix::Store;
-use Nix::CopyClosure;
-use List::Util qw(sum);
-
-binmode STDERR, ":encoding(utf8)";
-
-if (scalar @ARGV < 1) {
-    print STDERR <<EOF
-Usage: nix-copy-closure [--from | --to] HOSTNAME [--gzip] [--bzip2] [--xz] PATHS...
-EOF
-    ;
-    exit 1;
-}
-
-
-# Get the target host.
-my $sshHost;
-my $toMode = 1;
-my $includeOutputs = 0;
-my $dryRun = 0;
-my $useSubstitutes = 0;
-my $verbosity = 1;
-
-
-# !!! Copied from nix-pack-closure, should put this in a module.
-my @storePaths = ();
-
-while (@ARGV) {
-    my $arg = shift @ARGV;
-
-    if ($arg eq "--help") {
-        exec "man nix-copy-closure" or die;
-    }
-    elsif ($arg eq "--gzip" || $arg eq "--bzip2" || $arg eq "--xz") {
-        warn "$0: ‘$arg’ is not implemented\n" if $arg ne "--gzip";
-        push @globalSshOpts, "-C";
-    }
-    elsif ($arg eq "--from") {
-        $toMode = 0;
-    }
-    elsif ($arg eq "--to") {
-        $toMode = 1;
-    }
-    elsif ($arg eq "--include-outputs") {
-        $includeOutputs = 1;
-    }
-    elsif ($arg eq "--show-progress") {
-        warn "$0: ‘$arg’ is not implemented\n";
-    }
-    elsif ($arg eq "--dry-run") {
-        $dryRun = 1;
-    }
-    elsif ($arg eq "--use-substitutes" || $arg eq "-s") {
-        $useSubstitutes = 1;
-    }
-    elsif ($arg eq "-v") {
-        $verbosity++;
-        setVerbosity($verbosity);
-    }
-    elsif (!defined $sshHost) {
-        $sshHost = $arg;
-    }
-    else {
-        push @storePaths, $arg;
-    }
-}
-
-die "$0: you did not specify a host name\n" unless defined $sshHost;
-
-
-if ($toMode) { # Copy TO the remote machine.
-    Nix::CopyClosure::copyTo(
-        $sshHost, [ @storePaths ],
-        $includeOutputs, $dryRun, $useSubstitutes);
-}
-
-else { # Copy FROM the remote machine.
-
-    my ($from, $to) = connectToRemoteNix($sshHost, []);
-
-    # Query the closure of the given store paths on the remote
-    # machine.  Paths are assumed to be store paths; there is no
-    # resolution (following of symlinks).
-    syswrite($to, pack("L<x4L<x4", 7, $includeOutputs ? 1 : 0)) or die;
-    writeStrings(\@storePaths, $to);
-    my @missing = grep { !isValidPath($_) } readStrings($from);
-
-    # Export the store paths on the remote machine and import them locally.
-    if (scalar @missing > 0) {
-        print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n";
-        writeInt(5, $to); # == cmdExportPaths
-        writeInt(0, $to); # obsolete
-        writeStrings(\@missing, $to);
-        importPaths(fileno($from), 1);
-    }
-
-}
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 98ccc3ddc3..1ac9711a11 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -9,7 +9,6 @@
 #include "shared.hh"
 #include "pathlocks.hh"
 #include "globals.hh"
-#include "serve-protocol.hh"
 #include "serialise.hh"
 #include "store-api.hh"
 #include "derivations.hh"
diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk
index 05b8cb4514..62d5a010c2 100644
--- a/src/build-remote/local.mk
+++ b/src/build-remote/local.mk
@@ -8,4 +8,4 @@ build-remote_LIBS = libmain libutil libformat libstore
 
 build-remote_SOURCES := $(d)/build-remote.cc
 
-build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" -Isrc/nix-store
+build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\""
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 3d01594009..cce0458c69 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -39,6 +39,8 @@ private:
     string uri;
 
     Path key;
+
+    bool compress;
 };
 
 SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
@@ -48,6 +50,7 @@ SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
     , socketPath((Path) tmpDir + "/ssh.sock")
     , uri(std::move(uri))
     , key(get(params, "ssh-key", ""))
+    , compress(get(params, "compress", "") == "true")
 {
     /* open a connection and perform the handshake to verify all is well */
     connections->get();
@@ -90,11 +93,12 @@ ref<FSAccessor> SSHStore::getFSAccessor()
 ref<RemoteStore::Connection> SSHStore::openConnection()
 {
     if ((pid_t) sshMaster == -1) {
+        auto flags = compress ? "-NMCS" : "-NMS";
         sshMaster = startProcess([&]() {
             if (key.empty())
-                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), uri.c_str(), NULL);
+                execlp("ssh", "ssh", flags, socketPath.c_str(), uri.c_str(), NULL);
             else
-                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL);
+                execlp("ssh", "ssh", flags, socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL);
             throw SysError("starting ssh master");
         });
     }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 8fdd627715..c8ca00f006 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -4,6 +4,7 @@
 #include "util.hh"
 #include "nar-info-disk-cache.hh"
 #include "thread-pool.hh"
+#include "derivations.hh"
 
 #include <future>
 
@@ -699,8 +700,27 @@ std::list<ref<Store>> getDefaultSubstituters()
 }
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths)
-{
+void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute)
+{
+    if (substitute) {
+        /* Filter out .drv files (we don't want to build anything). */
+        PathSet paths2;
+        for (auto & path : storePaths)
+            if (!isDerivation(path)) paths2.insert(path);
+        unsigned long long downloadSize, narSize;
+        PathSet willBuild, willSubstitute, unknown;
+        to->queryMissing(PathSet(paths2.begin(), paths2.end()),
+            willBuild, willSubstitute, unknown, downloadSize, narSize);
+        /* FIXME: should use ensurePath(), but it only
+           does one path at a time. */
+        if (!willSubstitute.empty())
+            try {
+                to->buildPaths(willSubstitute);
+            } catch (Error & e) {
+                printMsg(lvlError, format("warning: %1%") % e.msg());
+            }
+    }
+
     std::string copiedLabel = "copied";
 
     logger->setExpected(copiedLabel, storePaths.size());
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index ec3bf5a6fd..30ee433bf0 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -625,7 +625,7 @@ void removeTempRoots();
 ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"));
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths);
+void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute = false);
 
 enum StoreType {
     tDaemon,
diff --git a/src/nix-copy-closure/local.mk b/src/nix-copy-closure/local.mk
new file mode 100644
index 0000000000..42bb34dd82
--- /dev/null
+++ b/src/nix-copy-closure/local.mk
@@ -0,0 +1,7 @@
+programs += nix-copy-closure
+
+nix-copy-closure_DIR := $(d)
+
+nix-copy-closure_LIBS = libmain libutil libformat libstore
+
+nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
new file mode 100755
index 0000000000..b7e997ca4b
--- /dev/null
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -0,0 +1,60 @@
+#include "shared.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+int main(int argc, char ** argv)
+{
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+        auto gzip = false;
+        auto toMode = true;
+        auto includeOutputs = false;
+        auto dryRun = false;
+        auto useSubstitutes = false;
+        auto sshHost = string{};
+        auto storePaths = PathSet{};
+        parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
+            if (*arg == "--help")
+                showManPage("nix-copy-closure");
+            else if (*arg == "--version")
+                printVersion("nix-copy-closure");
+            else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") {
+                if (*arg != "--gzip")
+                    printMsg(lvlError, format("Warning: ‘%1%’ is not implemented, falling back to gzip") % *arg);
+                gzip = true;
+            } else if (*arg == "--from")
+                toMode = false;
+            else if (*arg == "--to")
+                toMode = true;
+            else if (*arg == "--include-outputs")
+                includeOutputs = true;
+            else if (*arg == "--show-progress")
+                printMsg(lvlError, "Warning: ‘--show-progress’ is not implemented");
+            else if (*arg == "--dry-run")
+                dryRun = true;
+            else if (*arg == "--use-substitutes" || *arg == "-s")
+                useSubstitutes = true;
+            else if (sshHost.empty())
+                sshHost = *arg;
+            else
+                storePaths.insert(*arg);
+            return true;
+        });
+        if (sshHost.empty())
+            throw UsageError("no host name specified");
+
+        auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : "");
+        auto to = toMode ? openStore(remoteUri) : openStore();
+        auto from = toMode ? openStore() : openStore(remoteUri);
+        if (includeOutputs) {
+            auto newPaths = PathSet{};
+            for (const auto & p : storePaths) {
+                auto outputs = from->queryDerivationOutputs(p);
+                newPaths.insert(outputs.begin(), outputs.end());
+            }
+            storePaths.insert(newPaths.begin(), newPaths.end());
+        }
+        copyPaths(from, to, Paths(storePaths.begin(), storePaths.end()), useSubstitutes);
+    });
+}