diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/GeneratePatches.pm.in | 333 | ||||
| -rw-r--r-- | scripts/Makefile.am | 10 | ||||
| -rw-r--r-- | scripts/NixConfig.pm.in | 17 | ||||
| -rw-r--r-- | scripts/NixManifest.pm.in | 350 | ||||
| -rw-r--r-- | scripts/download-using-manifests.pl.in | 30 | ||||
| -rw-r--r-- | scripts/nix-generate-patches.in | 6 | ||||
| -rw-r--r-- | scripts/nix-install-package.in | 9 | ||||
| -rw-r--r-- | scripts/nix-prefetch-url.in | 17 | ||||
| -rw-r--r-- | scripts/nix-pull.in | 18 | ||||
| -rw-r--r-- | scripts/nix-push.in | 9 | ||||
| -rwxr-xr-x | scripts/update-manifest.pl | 52 |
11 files changed, 48 insertions, 803 deletions
diff --git a/scripts/GeneratePatches.pm.in b/scripts/GeneratePatches.pm.in deleted file mode 100644 index 4bb5b05a8c9e..000000000000 --- a/scripts/GeneratePatches.pm.in +++ /dev/null @@ -1,333 +0,0 @@ -use strict; -use File::Temp qw(tempdir); -use File::stat; - - -# Some patch generations options. - -# Max size of NAR archives to generate patches for. -my $maxNarSize = $ENV{"NIX_MAX_NAR_SIZE"}; -$maxNarSize = 160 * 1024 * 1024 if !defined $maxNarSize; - -# If patch is bigger than this fraction of full archive, reject. -my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"}; -$maxPatchFraction = 0.60 if !defined $maxPatchFraction; - -my $timeLimit = $ENV{"NIX_BSDIFF_TIME_LIMIT"}; -$timeLimit = 180 if !defined $timeLimit; - -my $hashAlgo = "sha256"; - - -sub findOutputPaths { - my $narFiles = shift; - - my %outPaths; - - foreach my $p (keys %{$narFiles}) { - - # Ignore derivations. - next if ($p =~ /\.drv$/); - - # Ignore builders (too much ambiguity -- they're all called - # `builder.sh'). - next if ($p =~ /\.sh$/); - next if ($p =~ /\.patch$/); - - # Don't bother including tar files etc. - next if ($p =~ /\.tar$/ || $p =~ /\.tar\.(gz|bz2|Z|lzma|xz)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/ || $p =~ /\.tgz$/ || $p =~ /\.rpm$/ || $p =~ /cvs-export$/ || $p =~ /fetchhg$/); - - $outPaths{$p} = 1; - } - - return %outPaths; -} - - -sub getNameVersion { - my $p = shift; - $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/; - my $name = $1; - my $version = $2; - return undef unless defined $name && defined $version; - $name =~ s/^-//; - $version =~ s/^-//; - return ($name, $version); -} - - -# A quick hack to get a measure of the `distance' between two -# versions: it's just the position of the first character that differs -# (or 999 if they are the same). -sub versionDiff { - my $s = shift; - my $t = shift; - my $i; - return 999 if $s eq $t; - for ($i = 0; $i < length $s; $i++) { - return $i if $i >= length $t or - substr($s, $i, 1) ne substr($t, $i, 1); - } - return $i; -} - - -sub getNarBz2 { - my $narPath = shift; - my $narFiles = shift; - my $storePath = shift; - - my $narFileList = $$narFiles{$storePath}; - die "missing path $storePath" unless defined $narFileList; - - my $narFile = @{$narFileList}[0]; - die unless defined $narFile; - - $narFile->{url} =~ /\/([^\/]+)$/; - die unless defined $1; - return "$narPath/$1"; -} - - -sub containsPatch { - my $patches = shift; - my $storePath = shift; - my $basePath = shift; - my $patchList = $$patches{$storePath}; - return 0 if !defined $patchList; - my $found = 0; - foreach my $patch (@{$patchList}) { - # !!! baseHash might differ - return 1 if $patch->{basePath} eq $basePath; - } - return 0; -} - - -sub generatePatches { - my ($srcNarFiles, $dstNarFiles, $srcPatches, $dstPatches, $narPath, $patchesPath, $patchesURL, $tmpDir) = @_; - - my %srcOutPaths = findOutputPaths $srcNarFiles; - my %dstOutPaths = findOutputPaths $dstNarFiles; - - # For each output path in the destination, see if we need to / can - # create a patch. - - print STDERR "creating patches...\n"; - - foreach my $p (keys %dstOutPaths) { - - # If exactly the same path already exists in the source, skip it. - next if defined $srcOutPaths{$p}; - - print " $p\n"; - - # If not, then we should find the paths in the source that are - # `most' likely to be present on a system that wants to - # install this path. - - (my $name, my $version) = getNameVersion $p; - next unless defined $name && defined $version; - - my @closest = (); - my $closestVersion; - my $minDist = -1; # actually, larger means closer - - # Find all source paths with the same name. - - foreach my $q (keys %srcOutPaths) { - (my $name2, my $version2) = getNameVersion $q; - next unless defined $name2 && defined $version2; - - if ($name eq $name2) { - - my $srcSystem = @{$$dstNarFiles{$p}}[0]->{system}; - my $dstSystem = @{$$srcNarFiles{$q}}[0]->{system}; - if (defined $srcSystem && defined $dstSystem && $srcSystem ne $dstSystem) { - print " SKIPPING $q due to different systems ($srcSystem vs. $dstSystem)\n"; - next; - } - - # If the sizes differ too much, then skip. This - # disambiguates between, e.g., a real component and a - # wrapper component (cf. Firefox in Nixpkgs). - my $srcSize = @{$$srcNarFiles{$q}}[0]->{size}; - my $dstSize = @{$$dstNarFiles{$p}}[0]->{size}; - my $ratio = $srcSize / $dstSize; - $ratio = 1 / $ratio if $ratio < 1; - # print " SIZE $srcSize $dstSize $ratio $q\n"; - - if ($ratio >= 3) { - print " SKIPPING $q due to size ratio $ratio ($srcSize vs. $dstSize)\n"; - next; - } - - # If there are multiple matching names, include the - # ones with the closest version numbers. - my $dist = versionDiff $version, $version2; - if ($dist > $minDist) { - $minDist = $dist; - @closest = ($q); - $closestVersion = $version2; - } elsif ($dist == $minDist) { - push @closest, $q; - } - } - } - - if (scalar(@closest) == 0) { - print " NO BASE: $p\n"; - next; - } - - foreach my $closest (@closest) { - - # Generate a patch between $closest and $p. - print STDERR " $p <- $closest\n"; - - # If the patch already exists, skip it. - if (containsPatch($srcPatches, $p, $closest) || - containsPatch($dstPatches, $p, $closest)) - { - print " skipping, already exists\n"; - next; - } - - my $srcNarBz2 = getNarBz2 $narPath, $srcNarFiles, $closest; - my $dstNarBz2 = getNarBz2 $narPath, $dstNarFiles, $p; - - if (! -f $srcNarBz2) { - warn "patch source archive $srcNarBz2 is missing\n"; - next; - } - - system("@bunzip2@ < $srcNarBz2 > $tmpDir/A") == 0 - or die "cannot unpack $srcNarBz2"; - - if (stat("$tmpDir/A")->size >= $maxNarSize) { - print " skipping, source is too large\n"; - next; - } - - system("@bunzip2@ < $dstNarBz2 > $tmpDir/B") == 0 - or die "cannot unpack $dstNarBz2"; - - if (stat("$tmpDir/B")->size >= $maxNarSize) { - print " skipping, destination is too large\n"; - next; - } - - my $time1 = time(); - my $res = system("ulimit -t $timeLimit; @libexecdir@/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF"); - my $time2 = time(); - if ($res) { - warn "binary diff computation aborted after ", $time2 - $time1, " seconds\n"; - next; - } - - my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/A` or die; - chomp $baseHash; - - my $narHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/B` or die; - chomp $narHash; - - my $narDiffHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/DIFF` or die; - chomp $narDiffHash; - - my $narDiffSize = stat("$tmpDir/DIFF")->size; - my $dstNarBz2Size = stat($dstNarBz2)->size; - - print " size $narDiffSize; full size $dstNarBz2Size; ", $time2 - $time1, " seconds\n"; - - if ($narDiffSize >= $dstNarBz2Size) { - print " rejecting; patch bigger than full archive\n"; - next; - } - - if ($narDiffSize / $dstNarBz2Size >= $maxPatchFraction) { - print " rejecting; patch too large relative to full archive\n"; - next; - } - - my $finalName = "$narDiffHash.nar-bsdiff"; - - if (-e "$patchesPath/$finalName") { - print " not copying, already exists\n"; - } - - else { - system("cp '$tmpDir/DIFF' '$patchesPath/$finalName.tmp'") == 0 - or die "cannot copy diff"; - rename("$patchesPath/$finalName.tmp", "$patchesPath/$finalName") - or die "cannot rename $patchesPath/$finalName.tmp"; - } - - # Add the patch to the manifest. - addPatch $dstPatches, $p, - { url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash" - , size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash" - , narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff" - }; - } - } -} - - -# Propagate useful patches from $srcPatches to $dstPatches. A patch -# is useful if it produces either paths in the $dstNarFiles or paths -# that can be used as the base for other useful patches. -sub propagatePatches { - my ($srcPatches, $dstNarFiles, $dstPatches) = @_; - - print STDERR "propagating patches...\n"; - - my $changed; - do { - # !!! we repeat this to reach the transitive closure; inefficient - $changed = 0; - - print STDERR "loop\n"; - - my %dstBasePaths; - foreach my $q (keys %{$dstPatches}) { - foreach my $patch (@{$$dstPatches{$q}}) { - $dstBasePaths{$patch->{basePath}} = 1; - } - } - - foreach my $p (keys %{$srcPatches}) { - my $patchList = $$srcPatches{$p}; - - my $include = 0; - - # Is path $p included in the destination? If so, include - # patches that produce it. - $include = 1 if defined $$dstNarFiles{$p}; - - # Is path $p a path that serves as a base for paths in the - # destination? If so, include patches that produce it. - # !!! check baseHash - $include = 1 if defined $dstBasePaths{$p}; - - if ($include) { - foreach my $patch (@{$patchList}) { - $changed = 1 if addPatch $dstPatches, $p, $patch; - } - } - - } - - } while $changed; -} - - -# Add all new patches in $srcPatches to $dstPatches. -sub copyPatches { - my ($srcPatches, $dstPatches) = @_; - foreach my $p (keys %{$srcPatches}) { - addPatch $dstPatches, $p, $_ foreach @{$$srcPatches{$p}}; - } -} - - -return 1; diff --git a/scripts/Makefile.am b/scripts/Makefile.am index 60bb0a9b8105..752fcd574025 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -3,20 +3,17 @@ bin_SCRIPTS = nix-collect-garbage \ nix-install-package nix-channel nix-build \ nix-copy-closure nix-generate-patches -noinst_SCRIPTS = nix-profile.sh GeneratePatches.pm \ +noinst_SCRIPTS = nix-profile.sh \ find-runtime-roots.pl build-remote.pl nix-reduce-build \ copy-from-other-stores.pl nix-http-export.cgi -nix-pull nix-push: NixManifest.pm NixConfig.pm download-using-manifests.pl +nix-pull nix-push: download-using-manifests.pl -install-exec-local: NixManifest.pm GeneratePatches.pm download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl +install-exec-local: download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl $(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d $(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh $(INSTALL) -d $(DESTDIR)$(libexecdir)/nix - $(INSTALL_DATA) NixManifest.pm $(DESTDIR)$(libexecdir)/nix - $(INSTALL_DATA) NixConfig.pm $(DESTDIR)$(libexecdir)/nix $(INSTALL_DATA) SSH.pm $(DESTDIR)$(libexecdir)/nix - $(INSTALL_DATA) GeneratePatches.pm $(DESTDIR)$(libexecdir)/nix $(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix $(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix $(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters @@ -30,7 +27,6 @@ EXTRA_DIST = nix-collect-garbage.in \ nix-pull.in nix-push.in nix-profile.sh.in \ nix-prefetch-url.in nix-install-package.in \ nix-channel.in \ - NixManifest.pm.in \ NixConfig.pm.in \ SSH.pm \ GeneratePatches.pm.in \ diff --git a/scripts/NixConfig.pm.in b/scripts/NixConfig.pm.in deleted file mode 100644 index aeb443aeed3a..000000000000 --- a/scripts/NixConfig.pm.in +++ /dev/null @@ -1,17 +0,0 @@ -use strict; - -sub readConfig { - my %config; - my $config = "@sysconfdir@/nix/nix.conf"; - return unless -f $config; - - open CONFIG, "<$config" or die "cannot open `$config'"; - while (<CONFIG>) { - /^\s*([\w|-]+)\s*=\s*(.*)$/ or next; - $config{$1} = $2; - print "|$1| -> |$2|\n"; - } - close CONFIG; -} - -return 1; diff --git a/scripts/NixManifest.pm.in b/scripts/NixManifest.pm.in deleted file mode 100644 index d080dcee7478..000000000000 --- a/scripts/NixManifest.pm.in +++ /dev/null @@ -1,350 +0,0 @@ -use strict; -use DBI; -use Cwd; -use File::stat; -use File::Path; -use Fcntl ':flock'; - - -sub addNAR { - my ($narFiles, $storePath, $info) = @_; - - $$narFiles{$storePath} = [] - unless defined $$narFiles{$storePath}; - - my $narFileList = $$narFiles{$storePath}; - - my $found = 0; - foreach my $narFile (@{$narFileList}) { - $found = 1 if $narFile->{url} eq $info->{url}; - } - - push @{$narFileList}, $info if !$found; -} - - -sub addPatch { - my ($patches, $storePath, $patch) = @_; - - $$patches{$storePath} = [] - unless defined $$patches{$storePath}; - - my $patchList = $$patches{$storePath}; - - my $found = 0; - foreach my $patch2 (@{$patchList}) { - $found = 1 if - $patch2->{url} eq $patch->{url} && - $patch2->{basePath} eq $patch->{basePath}; - } - - push @{$patchList}, $patch if !$found; - - return !$found; -} - - -sub readManifest_ { - my ($manifest, $addNAR, $addPatch) = @_; - - open MANIFEST, "<$manifest" - or die "cannot open `$manifest': $!"; - - my $inside = 0; - my $type; - - my $manifestVersion = 2; - - my ($storePath, $url, $hash, $size, $basePath, $baseHash, $patchType); - my ($narHash, $narSize, $references, $deriver, $copyFrom, $system); - - while (<MANIFEST>) { - chomp; - s/\#.*$//g; - next if (/^$/); - - if (!$inside) { - - if (/^\s*(\w*)\s*\{$/) { - $type = $1; - $type = "narfile" if $type eq ""; - $inside = 1; - undef $storePath; - undef $url; - undef $hash; - undef $size; - undef $narHash; - undef $narSize; - undef $basePath; - undef $baseHash; - undef $patchType; - undef $system; - $references = ""; - $deriver = ""; - } - - } else { - - if (/^\}$/) { - $inside = 0; - - if ($type eq "narfile") { - &$addNAR($storePath, - { url => $url, hash => $hash, size => $size - , narHash => $narHash, narSize => $narSize - , references => $references - , deriver => $deriver - , system => $system - }); - } - - elsif ($type eq "patch") { - &$addPatch($storePath, - { url => $url, hash => $hash, size => $size - , basePath => $basePath, baseHash => $baseHash - , narHash => $narHash, narSize => $narSize - , patchType => $patchType - }); - } - - } - - elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) { $storePath = $1; } - elsif (/^\s*CopyFrom:\s*(\/\S+)\s*$/) { $copyFrom = $1; } - elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; } - elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; } - elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; } - elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) { } # obsolete - elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; } - elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; } - elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; } - elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; } - elsif (/^\s*NarSize:\s*(\d+)\s*$/) { $narSize = $1; } - elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; } - elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; } - elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; } - elsif (/^\s*System:\s*(\S+)\s*$/) { $system = $1; } - - # Compatibility; - elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; } - elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = "md5:$1"; } - - } - } - - close MANIFEST; - - return $manifestVersion; -} - - -sub readManifest { - my ($manifest, $narFiles, $patches) = @_; - readManifest_($manifest, - sub { addNAR($narFiles, @_); }, - sub { addPatch($patches, @_); } ); -} - - -sub writeManifest { - my ($manifest, $narFiles, $patches, $noCompress) = @_; - - open MANIFEST, ">$manifest.tmp"; # !!! check exclusive - - print MANIFEST "version {\n"; - print MANIFEST " ManifestVersion: 3\n"; - print MANIFEST "}\n"; - - foreach my $storePath (sort (keys %{$narFiles})) { - my $narFileList = $$narFiles{$storePath}; - foreach my $narFile (@{$narFileList}) { - print MANIFEST "{\n"; - print MANIFEST " StorePath: $storePath\n"; - print MANIFEST " NarURL: $narFile->{url}\n"; - print MANIFEST " Hash: $narFile->{hash}\n" if defined $narFile->{hash}; - print MANIFEST " Size: $narFile->{size}\n" if defined $narFile->{size}; - print MANIFEST " NarHash: $narFile->{narHash}\n"; - print MANIFEST " NarSize: $narFile->{narSize}\n" if $narFile->{narSize}; - print MANIFEST " References: $narFile->{references}\n" - if defined $narFile->{references} && $narFile->{references} ne ""; - print MANIFEST " Deriver: $narFile->{deriver}\n" - if defined $narFile->{deriver} && $narFile->{deriver} ne ""; - print MANIFEST " System: $narFile->{system}\n" if defined $narFile->{system}; - print MANIFEST "}\n"; - } - } - - foreach my $storePath (sort (keys %{$patches})) { - my $patchList = $$patches{$storePath}; - foreach my $patch (@{$patchList}) { - print MANIFEST "patch {\n"; - print MANIFEST " StorePath: $storePath\n"; - print MANIFEST " NarURL: $patch->{url}\n"; - print MANIFEST " Hash: $patch->{hash}\n"; - print MANIFEST " Size: $patch->{size}\n"; - print MANIFEST " NarHash: $patch->{narHash}\n"; - print MANIFEST " NarSize: $patch->{narSize}\n" if $patch->{narSize}; - print MANIFEST " BasePath: $patch->{basePath}\n"; - print MANIFEST " BaseHash: $patch->{baseHash}\n"; - print MANIFEST " Type: $patch->{patchType}\n"; - print MANIFEST "}\n"; - } - } - - - close MANIFEST; - - rename("$manifest.tmp", $manifest) - or die "cannot rename $manifest.tmp: $!"; - - - # Create a bzipped manifest. - unless (defined $noCompress) { - system("@bzip2@ < $manifest > $manifest.bz2.tmp") == 0 - or die "cannot compress manifest"; - - rename("$manifest.bz2.tmp", "$manifest.bz2") - or die "cannot rename $manifest.bz2.tmp: $!"; - } -} - - -sub updateManifestDB { - my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "@localstatedir@/nix/manifests"); - - mkpath($manifestDir); - - my $dbPath = "$manifestDir/cache.sqlite"; - - # Open/create the database. - our $dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "") - or die "cannot open database `$dbPath'"; - $dbh->{RaiseError} = 1; - $dbh->{PrintError} = 0; - - $dbh->do("pragma foreign_keys = on"); - $dbh->do("pragma synchronous = off"); # we can always reproduce the cache - $dbh->do("pragma journal_mode = truncate"); - - # Initialise the database schema, if necessary. - $dbh->do(<<EOF); - create table if not exists Manifests ( - id integer primary key autoincrement not null, - path text unique not null, - timestamp integer not null - ); -EOF - - $dbh->do(<<EOF); - create table if not exists NARs ( - id integer primary key autoincrement not null, - manifest integer not null, - storePath text not null, - url text not null, - hash text, - size integer, - narHash text, - narSize integer, - refs text, - deriver text, - system text, - foreign key (manifest) references Manifests(id) on delete cascade - ); -EOF - - $dbh->do("create index if not exists NARs_storePath on NARs(storePath)"); - - $dbh->do(<<EOF); - create table if not exists Patches ( - id integer primary key autoincrement not null, - manifest integer not null, - storePath text not null, - basePath text not null, - baseHash text not null, - url text not null, - hash text, - size integer, - narHash text, - narSize integer, - patchType text not null, - foreign key (manifest) references Manifests(id) on delete cascade - ); -EOF - - $dbh->do("create index if not exists Patches_storePath on Patches(storePath)"); - - # Acquire an exclusive lock to ensure that only one process - # updates the DB at the same time. This isn't really necessary, - # but it prevents work duplication and lock contention in SQLite. - open MAINLOCK, ">>$manifestDir/cache.lock" or die; - flock(MAINLOCK, LOCK_EX) or die; - - $dbh->begin_work; - - # Read each manifest in $manifestDir and add it to the database, - # unless we've already done so on a previous run. - my %seen; - - for my $manifest (glob "$manifestDir/*.nixmanifest") { - $manifest = Cwd::abs_path($manifest); - my $timestamp = lstat($manifest)->mtime; - $seen{$manifest} = 1; - - next if scalar @{$dbh->selectcol_arrayref( - "select 1 from Manifests where path = ? and timestamp = ?", - {}, $manifest, $timestamp)} == 1; - - print STDERR "caching $manifest...\n"; - - $dbh->do("delete from Manifests where path = ?", {}, $manifest); - - $dbh->do("insert into Manifests(path, timestamp) values (?, ?)", - {}, $manifest, $timestamp); - - our $id = $dbh->last_insert_id("", "", "", ""); - - sub addNARToDB { - my ($storePath, $narFile) = @_; - $dbh->do( - "insert into NARs(manifest, storePath, url, hash, size, narHash, " . - "narSize, refs, deriver, system) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - {}, $id, $storePath, $narFile->{url}, $narFile->{hash}, $narFile->{size}, - $narFile->{narHash}, $narFile->{narSize}, $narFile->{references}, - $narFile->{deriver}, $narFile->{system}); - }; - - sub addPatchToDB { - my ($storePath, $patch) = @_; - $dbh->do( - "insert into Patches(manifest, storePath, basePath, baseHash, url, hash, " . - "size, narHash, narSize, patchType) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - {}, $id, $storePath, $patch->{basePath}, $patch->{baseHash}, $patch->{url}, - $patch->{hash}, $patch->{size}, $patch->{narHash}, $patch->{narSize}, - $patch->{patchType}); - }; - - my $version = readManifest_($manifest, \&addNARToDB, \&addPatchToDB); - - if ($version < 3) { - die "you have an old-style manifest `$manifest'; please delete it"; - } - if ($version >= 10) { - die "manifest `$manifest' is too new; please delete it or upgrade Nix"; - } - } - - # Removed cached information for removed manifests from the DB. - foreach my $manifest (@{$dbh->selectcol_arrayref("select path from Manifests")}) { - next if defined $seen{$manifest}; - $dbh->do("delete from Manifests where path = ?", {}, $manifest); - } - - $dbh->commit; - - close MAINLOCK; - - return $dbh; -} - - -return 1; diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in index ca4d97b51aa6..a827a995f919 100644 --- a/scripts/download-using-manifests.pl.in +++ b/scripts/download-using-manifests.pl.in @@ -1,16 +1,14 @@ -#! @perl@ -w -I@libexecdir@/nix @perlFlags@ +#! @perl@ -w @perlFlags@ use strict; -use NixManifest; +use Nix::Config; +use Nix::Manifest; use POSIX qw(strftime); use File::Temp qw(tempdir); -my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@"; - STDOUT->autoflush(1); -my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "@localstatedir@/nix/manifests"); -my $logFile = "@localstatedir@/log/nix/downloads"; +my $logFile = "$Nix::Config::logDir/downloads"; # For queries, skip expensive calls to nix-hash etc. We're just # estimating the expected download size. @@ -26,7 +24,7 @@ sub isValidPath { if ($fast) { return -e $p; } else { - return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0; + return system("$Nix::Config::binDir/nix-store --check-validity '$p' 2> /dev/null") == 0; } } @@ -108,8 +106,8 @@ sub computeSmallestDownload { my $format = "--base32"; $format = "" if $baseHashAlgo eq "md5"; my $hash = $fast && $baseHashAlgo eq "sha256" - ? `$binDir/nix-store -q --hash "$patch->{basePath}"` - : `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`; + ? `$Nix::Config::binDir/nix-store -q --hash "$patch->{basePath}"` + : `$Nix::Config::binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`; chomp $hash; $hash =~ s/.*://; next if $hash ne $baseHash; @@ -282,7 +280,7 @@ sub downloadFile { my $url = shift; $ENV{"PRINT_PATH"} = 1; $ENV{"QUIET"} = 1; - my ($hash, $path) = `$binDir/nix-prefetch-url '$url'`; + my ($hash, $path) = `$Nix::Config::binDir/nix-prefetch-url '$url'`; die "download of `$url' failed" . ($! ? ": $!" : "") unless $? == 0; chomp $path; return $path; @@ -306,7 +304,7 @@ while (scalar @path > 0) { # as a base to one or more patches. So turn the base path # into a NAR archive, to which we can apply the patch. print " packing base path...\n"; - system("$binDir/nix-store --dump $v > $tmpNar") == 0 + system("$Nix::Config::binDir/nix-store --dump $v > $tmpNar") == 0 or die "cannot dump `$v'"; } } @@ -324,7 +322,7 @@ while (scalar @path > 0) { # Apply the patch to the NAR archive produced in step 1 (for # the already present path) or a later step (for patch sequences). print " applying patch...\n"; - system("@libexecdir@/bspatch $tmpNar $tmpNar2 $patchPath") == 0 + system("$Nix::Config::libexecDir/bspatch $tmpNar $tmpNar2 $patchPath") == 0 or die "cannot apply patch `$patchPath' to $tmpNar"; if ($curStep < $maxStep) { @@ -334,7 +332,7 @@ while (scalar @path > 0) { # This was the last patch. Unpack the final NAR archive # into the target path. print " unpacking patched archive...\n"; - system("$binDir/nix-store --restore $v < $tmpNar2") == 0 + system("$Nix::Config::binDir/nix-store --restore $v < $tmpNar2") == 0 or die "cannot unpack $tmpNar2 into `$v'"; } @@ -354,12 +352,12 @@ while (scalar @path > 0) { if ($curStep < $maxStep) { # The archive will be used a base to a patch. - system("@bunzip2@ < '$narFilePath' > $tmpNar") == 0 + system("$Nix::Config::bzip2 -d < '$narFilePath' > $tmpNar") == 0 or die "cannot unpack `$narFilePath' into `$v'"; } else { # Unpack the archive into the target path. print " unpacking archive...\n"; - system("@bunzip2@ < '$narFilePath' | $binDir/nix-store --restore '$v'") == 0 + system("$Nix::Config::bzip2 -d < '$narFilePath' | $Nix::Config::binDir/nix-store --restore '$v'") == 0 or die "cannot unpack `$narFilePath' into `$v'"; } @@ -382,7 +380,7 @@ if (defined $finalNarHash) { ($hashAlgo eq "sha256" && length($hash) != 64) ? "--base32" : ""; - my $hash2 = `@bindir@/nix-hash --type $hashAlgo $extraFlag $targetPath` + my $hash2 = `$Nix::Config::binDir/nix-hash --type $hashAlgo $extraFlag $targetPath` or die "cannot compute hash of path `$targetPath'"; chomp $hash2; diff --git a/scripts/nix-generate-patches.in b/scripts/nix-generate-patches.in index 1f32ab410920..4cb843382b52 100644 --- a/scripts/nix-generate-patches.in +++ b/scripts/nix-generate-patches.in @@ -1,9 +1,9 @@ -#! @perl@ -w -I@libexecdir@/nix @perlFlags@ +#! @perl@ -w @perlFlags@ use strict; use File::Temp qw(tempdir); -use NixManifest; -use GeneratePatches; +use Nix::Manifest; +use Nix::GeneratePatches; if (scalar @ARGV != 5) { print STDERR <<EOF; diff --git a/scripts/nix-install-package.in b/scripts/nix-install-package.in index f40cfc7d0b23..178b14c7468f 100644 --- a/scripts/nix-install-package.in +++ b/scripts/nix-install-package.in @@ -2,8 +2,7 @@ use strict; use File::Temp qw(tempdir); - -my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@"; +use Nix::Config; sub usageError { @@ -61,7 +60,7 @@ if ($interactive && !defined $ENV{"NIX_HAVE_TERMINAL"}) { $ENV{"NIX_HAVE_TERMINAL"} = "1"; $ENV{"LD_LIBRARY_PATH"} = ""; foreach my $term ("xterm", "konsole", "gnome-terminal", "xterm") { - exec($term, "-e", "$binDir/nix-install-package", @ARGV); + exec($term, "-e", "$Nix::Config::binDir/nix-install-package", @ARGV); } die "cannot execute `xterm'"; } @@ -132,12 +131,12 @@ $ENV{NIX_REMOTE} = ""; print "\nPulling manifests...\n"; -system("$binDir/nix-pull", $manifestURL) == 0 +system("$Nix::Config::binDir/nix-pull", $manifestURL) == 0 or barf "nix-pull failed: $?"; print "\nInstalling package...\n"; -system("$binDir/nix-env", "--install", $outPath, "--force-name", $drvName, @extraNixEnvArgs) == 0 +system("$Nix::Config::binDir/nix-env", "--install", $outPath, "--force-name", $drvName, @extraNixEnvArgs) == 0 or barf "nix-env failed: $?"; diff --git a/scripts/nix-prefetch-url.in b/scripts/nix-prefetch-url.in index 31170fa953ea..45bad75f3e9f 100644 --- a/scripts/nix-prefetch-url.in +++ b/scripts/nix-prefetch-url.in @@ -3,6 +3,9 @@ url=$1 expHash=$2 +binDir=@bindir@ +if [ -n "$NIX_BIN_DIR" ]; then binDir="$NIX_BIN_DIR"; fi + # needed to make it work on NixOS export PATH=$PATH:@coreutils@ @@ -31,8 +34,8 @@ if test -z "$name"; then echo "invalid url"; exit 1; fi # If the hash was given, a file with that hash may already be in the # store. if test -n "$expHash"; then - finalPath=$(@bindir@/nix-store --print-fixed-path "$hashType" "$expHash" "$name") - if ! @bindir@/nix-store --check-validity "$finalPath" 2> /dev/null; then + finalPath=$($binDir/nix-store --print-fixed-path "$hashType" "$expHash" "$name") + if ! $bindir/nix-store --check-validity "$finalPath" 2> /dev/null; then finalPath= fi hash=$expHash @@ -103,7 +106,7 @@ if test -z "$finalPath"; then # garbage-collected independently. if test -n "$NIX_DOWNLOAD_CACHE"; then echo -n "$url" > $tmpPath/url - urlHash=$(@bindir@/nix-hash --type sha256 --base32 --flat $tmpPath/url) + urlHash=$($binDir/nix-hash --type sha256 --base32 --flat $tmpPath/url) echo "$url" > "$NIX_DOWNLOAD_CACHE/$urlHash.url" cachedHashFN="$NIX_DOWNLOAD_CACHE/$urlHash.$hashType" cachedTimestampFN="$NIX_DOWNLOAD_CACHE/$urlHash.stamp" @@ -121,8 +124,8 @@ if test -z "$finalPath"; then # Curl didn't create $tmpFile, so apparently there's no newer # file on the server. hash=$(cat $cachedHashFN) - finalPath=$(@bindir@/nix-store --print-fixed-path "$hashType" "$hash" "$name") - if ! @bindir@/nix-store --check-validity "$finalPath" 2> /dev/null; then + finalPath=$($binDir/nix-store --print-fixed-path "$hashType" "$hash" "$name") + if ! $binDir/nix-store --check-validity "$finalPath" 2> /dev/null; then echo "cached contents of \`$url' disappeared, redownloading..." >&2 finalPath= cacheFlags="--remote-time" @@ -133,7 +136,7 @@ if test -z "$finalPath"; then if test -z "$finalPath"; then # Compute the hash. - hash=$(@bindir@/nix-hash --type "$hashType" $hashFormat --flat $tmpFile) + hash=$($binDir/nix-hash --type "$hashType" $hashFormat --flat $tmpFile) if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi if test -n "$NIX_DOWNLOAD_CACHE"; then @@ -142,7 +145,7 @@ if test -z "$finalPath"; then fi # Add the downloaded file to the Nix store. - finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile) + finalPath=$($binDir/nix-store --add-fixed "$hashType" $tmpFile) if test -n "$expHash" -a "$expHash" != "$hash"; then echo "hash mismatch for URL \`$url'" >&2 diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in index 51c7e681a5a7..8fb256179c02 100644 --- a/scripts/nix-pull.in +++ b/scripts/nix-pull.in @@ -1,17 +1,17 @@ -#! @perl@ -w -I@libexecdir@/nix @perlFlags@ +#! @perl@ -w @perlFlags@ use strict; use File::Temp qw(tempdir); -use NixManifest; +use Nix::Config; +use Nix::Manifest; my $tmpDir = tempdir("nix-pull.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die "cannot create a temporary directory"; -my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@"; my $libexecDir = ($ENV{"NIX_LIBEXEC_DIR"} or "@libexecdir@"); my $storeDir = ($ENV{"NIX_STORE_DIR"} or "@storedir@"); my $stateDir = ($ENV{"NIX_STATE_DIR"} or "@localstatedir@/nix"); -my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "$stateDir/manifests"); +my $manifestDir = $Nix::Config::manifestDir; # Prevent access problems in shared-stored installations. @@ -42,7 +42,7 @@ sub downloadFile { my $url = shift; $ENV{"PRINT_PATH"} = 1; $ENV{"QUIET"} = 1; - my ($dummy, $path) = `$binDir/nix-prefetch-url '$url'`; + my ($dummy, $path) = `$Nix::Config::binDir/nix-prefetch-url '$url'`; die "cannot fetch `$url'" if $? != 0; die "nix-prefetch-url did not return a path" unless defined $path; chomp $path; @@ -57,16 +57,16 @@ sub processURL { my $manifest; # First see if a bzipped manifest is available. - if (system("@curl@ --fail --silent --head '$url'.bz2 > /dev/null") == 0) { + if (system("$Nix::Config::curl --fail --silent --head '$url'.bz2 > /dev/null") == 0) { print "fetching list of Nix archives at `$url.bz2'...\n"; my $bzipped = downloadFile "$url.bz2"; $manifest = "$tmpDir/MANIFEST"; - system("@bunzip2@ < $bzipped > $manifest") == 0 + system("$Nix::Config::bzip2 -d < $bzipped > $manifest") == 0 or die "cannot decompress manifest"; - $manifest = (`$binDir/nix-store --add $manifest` + $manifest = (`$Nix::Config::binDir/nix-store --add $manifest` or die "cannot copy $manifest to the store"); chomp $manifest; } @@ -96,7 +96,7 @@ sub processURL { $baseName = $1; } - my $hash = `$binDir/nix-hash --flat '$manifest'` + my $hash = `$Nix::Config::binDir/nix-hash --flat '$manifest'` or die "cannot hash `$manifest'"; chomp $hash; diff --git a/scripts/nix-push.in b/scripts/nix-push.in index fd1ec21485e2..dcdad5721265 100644 --- a/scripts/nix-push.in +++ b/scripts/nix-push.in @@ -1,9 +1,10 @@ -#! @perl@ -w -I@libexecdir@/nix @perlFlags@ +#! @perl@ -w @perlFlags@ use strict; use File::Temp qw(tempdir); use File::stat; -use NixManifest; +use Nix::Config; +use Nix::Manifest; my $hashAlgo = "sha256"; @@ -13,7 +14,7 @@ my $tmpDir = tempdir("nix-push.XXXXXX", CLEANUP => 1, TMPDIR => 1) my $nixExpr = "$tmpDir/create-nars.nix"; my $manifest = "$tmpDir/MANIFEST"; -my $curl = "@curl@ --fail --silent"; +my $curl = "$Nix::Config::curl --fail --silent"; my $extraCurlFlags = ${ENV{'CURL_FLAGS'}}; $curl = "$curl $extraCurlFlags" if defined $extraCurlFlags; @@ -107,7 +108,7 @@ foreach my $storePath (@storePaths) { # Construct a Nix expression that creates a Nix archive. my $nixexpr = "((import $dataDir/nix/corepkgs/nar/nar.nix) " . - "{storePath = builtins.storePath \"$storePath\"; system = \"@system@\"; hashAlgo = \"$hashAlgo\";}) "; + "{ storePath = builtins.storePath \"$storePath\"; system = \"@system@\"; hashAlgo = \"$hashAlgo\"; }) "; print NIX $nixexpr; } diff --git a/scripts/update-manifest.pl b/scripts/update-manifest.pl deleted file mode 100755 index 566f6467328d..000000000000 --- a/scripts/update-manifest.pl +++ /dev/null @@ -1,52 +0,0 @@ -#! /usr/bin/perl -w -I. - -use strict; -use readmanifest; - -die unless scalar @ARGV == 2; - -my $cache = $ARGV[0]; -my $manifest = $ARGV[1]; -my %narFiles; -my %patches; - -readManifest $manifest, \%narFiles, \%patches; - -foreach my $storePath (keys %narFiles) { - my $narFileList = $narFiles{$storePath}; - - foreach my $narFile (@{$narFileList}) { - if (!defined $narFile->{size} or - !defined $narFile->{narHash}) - { - $narFile->{url} =~ /\/([^\/]+)$/; - die unless defined $1; - my $fn = "$cache/$1"; - - my @info = stat $fn or die; - $narFile->{size} = $info[7]; - - my $narHash; - my $hashFile = "$fn.NARHASH"; - if (-e $hashFile) { - open HASH, "<$hashFile" or die; - $narHash = <HASH>; - close HASH; - } else { - print "$fn\n"; - $narHash = `bunzip2 < '$fn' | nix-hash --flat /dev/stdin` or die; - open HASH, ">$hashFile" or die; - print HASH $narHash; - close HASH; - } - chomp $narHash; - $narFile->{narHash} = $narHash; - } - } -} - -if (! -e "$manifest.backup") { - system "mv --reply=no '$manifest' '$manifest.backup'"; -} - -writeManifest $manifest, \%narFiles, \%patches; |