1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
|
package Nix::CopyClosure;
use strict;
use Nix::Config;
use Nix::Store;
use List::Util qw(sum);
use IPC::Open2;
sub readInt {
my ($from) = @_;
my $resp;
sysread($from, $resp, 8) == 8 or die "did not receive valid reply from remote host\n";
return unpack("L<x4", $resp);
}
sub writeString {
my ($s, $to) = @_;
my $len = length $s;
my $req .= pack("L<x4", $len);
$req .= $s;
$req .= "\000" x (8 - $len % 8) if $len % 8;
syswrite($to, $req) or die;
}
sub copyTo {
my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor,
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_;
$useSubstitutes = 0 if $dryRun;
# Get the closure of this path.
my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs,
map { followLinksToStorePath $_ } @{$storePaths})));
# Start ‘nix-store --serve’ on the remote host.
my ($from, $to);
my $pid = open2($from, $to, "ssh $sshHost @{$sshOpts} nix-store --serve --write");
# Do the handshake.
eval {
my $SERVE_MAGIC_1 = 0x390c9deb; # FIXME
my $clientVersion = 0x200;
syswrite($to, pack("L<x4L<x4", $SERVE_MAGIC_1, $clientVersion)) or die;
die "did not get valid handshake from remote host\n" if readInt($from) != 0x5452eecb;
my $serverVersion = readInt($from);
die "unsupported server version\n" if $serverVersion < 0x200 || $serverVersion >= 0x300;
};
if ($@) {
chomp $@;
warn "$@; falling back to old closure copying method\n";
return oldCopyTo(\@closure, @_);
}
# Send the "query valid paths" command with the "lock" option
# enabled. This prevents a race where the remote host
# garbage-collect paths that are already there. Optionally, ask
# the remote host to substitute missing paths.
syswrite($to, pack("L<x4L<x4L<x4L<x4", 1, 1, $useSubstitutes, scalar @closure)) or die;
writeString($_, $to) foreach @closure;
# Get back the set of paths that are already valid on the remote host.
my %present;
my $n = readInt($from);
while ($n--) {
my $len = readInt($from);
my $s;
sysread($from, $s, $len) == $len or die;
$present{$s} = 1;
sysread($from, $s, 8 - $len % 8) if $len % 8; # skip padding
}
my @missing = grep { !$present{$_} } @closure;
return if !@missing;
my $missingSize = 0;
$missingSize += (queryPathInfo($_, 1))[3] foreach @missing;
printf STDERR "copying %d missing paths (%.2f MiB) to ‘$sshHost’...\n",
scalar(@missing), $missingSize / (1024.0^2);
return if $dryRun;
# Send the "import paths" command.
syswrite($to, pack("L<x4", 4)) or die;
writeString($compressor, $to);
if ($compressor || $progressViewer) {
# Compute the size of the closure for the progress viewer.
$progressViewer = "$progressViewer -s $missingSize" if $progressViewer;
# Start the compressor and/or progress viewer in between us
# and the remote host.
my $to_;
my $pid2 = open2(">&" . fileno($to), $to_,
$progressViewer && $compressor ? "$progressViewer | $compressor" : $progressViewer || $compressor);
close $to;
exportPaths(fileno($to_), $sign, @missing);
close $to_;
waitpid $pid2, 0;
} else {
exportPaths(fileno($to), $sign, @missing);
close $to;
}
readInt($from) == 1 or die "remote machine \`$sshHost' failed to import closure\n";
}
# For backwards compatibility with Nix <= 1.7. Will be removed
# eventually.
sub oldCopyTo {
my ($closure, $sshHost, $sshOpts, $storePaths, $compressor, $decompressor,
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_;
# Optionally use substitutes on the remote host.
if (!$dryRun && $useSubstitutes) {
system "ssh $sshHost @{$sshOpts} nix-store -r --ignore-unknown @$closure";
# Ignore exit status because this is just an optimisation.
}
# Ask the remote host which paths are invalid. Because of limits
# to the command line length, do this in chunks. Eventually,
# we'll want to use ‘--from-stdin’, but we can't rely on the
# target having this option yet.
my @missing;
my $missingSize = 0;
while (scalar(@$closure) > 0) {
my @ps = splice(@$closure, 0, 1500);
open(READ, "set -f; ssh $sshHost @{$sshOpts} nix-store --check-validity --print-invalid @ps|");
while (<READ>) {
chomp;
push @missing, $_;
my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($_, 1);
$missingSize += $narSize;
}
close READ or die;
}
$compressor = "$compressor |" if $compressor ne "";
$decompressor = "$decompressor |" if $decompressor ne "";
$progressViewer = "$progressViewer -s $missingSize |" if $progressViewer ne "";
# Export the store paths and import them on the remote machine.
if (scalar @missing > 0) {
print STDERR "copying ", scalar @missing, " missing paths to ‘$sshHost’...\n";
unless ($dryRun) {
open SSH, "| $progressViewer $compressor ssh $sshHost @{$sshOpts} '$decompressor nix-store --import' > /dev/null" or die;
exportPaths(fileno(SSH), $sign, @missing);
close SSH or die "copying store paths to remote machine `$sshHost' failed: $?";
}
}
}
1;
|
