From a5952405d2803ae0d29955fe6725cd9195327a07 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Nov 2011 11:37:40 +0000 Subject: * Re-use prepared statements across insertions into the manifest cache DB. This speeds up creating the cache from 16.1s to 7.9s on my system. --- perl/lib/Nix/Manifest.pm | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'perl/lib/Nix/Manifest.pm') diff --git a/perl/lib/Nix/Manifest.pm b/perl/lib/Nix/Manifest.pm index 7790cfe3b98c..f042e1f88db3 100644 --- a/perl/lib/Nix/Manifest.pm +++ b/perl/lib/Nix/Manifest.pm @@ -286,6 +286,14 @@ EOF open MAINLOCK, ">>$lockFile" or die "unable to acquire lock ‘$lockFile’: $!\n"; flock(MAINLOCK, LOCK_EX) or die; + our $insertNAR = $dbh->prepare( + "insert into NARs(manifest, storePath, url, hash, size, narHash, " . + "narSize, refs, deriver, system) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") or die; + + our $insertPatch = $dbh->prepare( + "insert into Patches(manifest, storePath, basePath, baseHash, url, hash, " . + "size, narHash, narSize, patchType) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + $dbh->begin_work; # Read each manifest in $manifestDir and add it to the database, @@ -312,20 +320,16 @@ EOF sub addNARToDB { my ($storePath, $narFile) = @_; - $dbh->do( - "insert into NARs(manifest, storePath, url, hash, size, narHash, " . - "narSize, refs, deriver, system) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - {}, $id, $storePath, $narFile->{url}, $narFile->{hash}, $narFile->{size}, + $insertNAR->execute( + $id, $storePath, $narFile->{url}, $narFile->{hash}, $narFile->{size}, $narFile->{narHash}, $narFile->{narSize}, $narFile->{references}, $narFile->{deriver}, $narFile->{system}); }; sub addPatchToDB { my ($storePath, $patch) = @_; - $dbh->do( - "insert into Patches(manifest, storePath, basePath, baseHash, url, hash, " . - "size, narHash, narSize, patchType) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - {}, $id, $storePath, $patch->{basePath}, $patch->{baseHash}, $patch->{url}, + $insertPatch->execute( + $id, $storePath, $patch->{basePath}, $patch->{baseHash}, $patch->{url}, $patch->{hash}, $patch->{size}, $patch->{narHash}, $patch->{narSize}, $patch->{patchType}); }; -- cgit 1.4.1 From 63ee5e4d2a46c3619c59307d7dbb08f25d32c3e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Nov 2011 11:56:19 +0000 Subject: * Remove obsolete line. --- perl/lib/Nix/Manifest.pm | 1 - 1 file changed, 1 deletion(-) (limited to 'perl/lib/Nix/Manifest.pm') diff --git a/perl/lib/Nix/Manifest.pm b/perl/lib/Nix/Manifest.pm index f042e1f88db3..21c61d284faf 100644 --- a/perl/lib/Nix/Manifest.pm +++ b/perl/lib/Nix/Manifest.pm @@ -120,7 +120,6 @@ sub readManifest_ { elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; } elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; } elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; } - elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) { } # obsolete elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; } elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; } elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; } -- cgit 1.4.1 From d7d7910ba48d898bda2db92a4f16a6179c855f7d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Nov 2011 16:25:38 +0000 Subject: * Don't decompress the manifests in /nix/var/nix/manifest. This saves disk space, and, since they're typically only decompressed once (to fill the manifest cache), doesn't make things slower. --- perl/lib/Nix/Manifest.pm | 10 ++++++++-- scripts/nix-pull.in | 11 +---------- 2 files changed, 9 insertions(+), 12 deletions(-) (limited to 'perl/lib/Nix/Manifest.pm') diff --git a/perl/lib/Nix/Manifest.pm b/perl/lib/Nix/Manifest.pm index 21c61d284faf..9c891abb8bd9 100644 --- a/perl/lib/Nix/Manifest.pm +++ b/perl/lib/Nix/Manifest.pm @@ -53,8 +53,14 @@ sub addPatch { sub readManifest_ { my ($manifest, $addNAR, $addPatch) = @_; - open MANIFEST, "<$manifest" - or die "cannot open `$manifest': $!"; + # Decompress the manifest if necessary. + if ($manifest =~ /\.bz2$/) { + open MANIFEST, "$Nix::Config::bzip2 -d < $manifest |" + or die "cannot decompress `$manifest': $!"; + } else { + open MANIFEST, "<$manifest" + or die "cannot open `$manifest': $!"; + } my $inside = 0; my $type; diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in index f3cba0c02619..d8352b2774bb 100755 --- a/scripts/nix-pull.in +++ b/scripts/nix-pull.in @@ -59,16 +59,7 @@ sub processURL { # First see if a bzipped manifest is available. if (system("$Nix::Config::curl --fail --silent --head '$url'.bz2 > /dev/null") == 0) { print "fetching list of Nix archives at `$url.bz2'...\n"; - my $bzipped = downloadFile "$url.bz2"; - - $manifest = "$tmpDir/MANIFEST"; - - system("$Nix::Config::bzip2 -d < $bzipped > $manifest") == 0 - or die "cannot decompress manifest"; - - $manifest = (`$Nix::Config::binDir/nix-store --add $manifest` - or die "cannot copy $manifest to the store"); - chomp $manifest; + $manifest = downloadFile "$url.bz2"; } # Otherwise, just get the uncompressed manifest. -- cgit 1.4.1 From f8e609c3413e38d403d986020079f24a2b82c063 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Nov 2011 16:41:48 +0000 Subject: * nix-pull: update the Nix manifest cache if necessary. Also, don't read the manifest just to check the version and print the number of paths. This makes nix-pull very fast for the cached cache (speeding up nixos-rebuild without the ‘--no-pull’ or ‘--fast’ options). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- perl/lib/Nix/Manifest.pm | 8 ++++---- scripts/nix-pull.in | 24 +++--------------------- 2 files changed, 7 insertions(+), 25 deletions(-) (limited to 'perl/lib/Nix/Manifest.pm') diff --git a/perl/lib/Nix/Manifest.pm b/perl/lib/Nix/Manifest.pm index 9c891abb8bd9..d1717a0a8ebd 100644 --- a/perl/lib/Nix/Manifest.pm +++ b/perl/lib/Nix/Manifest.pm @@ -305,8 +305,8 @@ EOF # unless we've already done so on a previous run. my %seen; - for my $manifest (glob "$manifestDir/*.nixmanifest") { - $manifest = Cwd::abs_path($manifest); + for my $manifestLink (glob "$manifestDir/*.nixmanifest") { + my $manifest = Cwd::abs_path($manifestLink); my $timestamp = lstat($manifest)->mtime; $seen{$manifest} = 1; @@ -342,10 +342,10 @@ EOF my $version = readManifest_($manifest, \&addNARToDB, \&addPatchToDB); if ($version < 3) { - die "you have an old-style manifest `$manifest'; please delete it"; + die "you have an old-style or corrupt manifest `$manifestLink'; please delete it"; } if ($version >= 10) { - die "manifest `$manifest' is too new; please delete it or upgrade Nix"; + die "manifest `$manifestLink' is too new; please delete it or upgrade Nix"; } } diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in index d8352b2774bb..74545a35004c 100755 --- a/scripts/nix-pull.in +++ b/scripts/nix-pull.in @@ -33,10 +33,6 @@ if (! -l $manifestDirLink) { # Process the URLs specified on the command line. -my %narFiles; -my %patches; - -my $skipWrongStore = 0; sub downloadFile { my $url = shift; @@ -68,20 +64,6 @@ sub processURL { $manifest = downloadFile $url; } - my $version = readManifest($manifest, \%narFiles, \%patches); - - die "`$url' is not a manifest or it is too old (i.e., for Nix <= 0.7)\n" if $version < 3; - die "manifest `$url' is too new\n" if $version >= 5; - - if ($skipWrongStore) { - foreach my $path (keys %narFiles) { - if (substr($path, 0, length($storeDir) + 1) ne "$storeDir/") { - print STDERR "warning: manifest `$url' assumes a Nix store at a different location than $storeDir, skipping...\n"; - exit 0; - } - } - } - my $baseName = "unnamed"; if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component $baseName = $1; @@ -120,12 +102,12 @@ sub processURL { while (@ARGV) { my $url = shift @ARGV; if ($url eq "--skip-wrong-store") { - $skipWrongStore = 1; + # No-op, no longer supported. } else { processURL $url; } } -my $size = scalar (keys %narFiles); -print "$size store paths in manifest\n"; +# Update the cache. +updateManifestDB(); -- cgit 1.4.1