* Fix concurrency issues in download-using-manifests' handling of the

  SQLite manifest cache.  The DBI AutoCommit feature caused every
  process to have an active transaction at all times, which could
  indefinitely block processes wanting to update the manifest cache.

* Disable fsync() in the manifest cache because we don't need
  integrity (the cache can always be recreated if it gets corrupted).

2 files changed, 12 insertions, 1 deletions

diff --git a/scripts/NixManifest.pm.in b/scripts/NixManifest.pm.in
index 2d4b626570fb..e1a0b97094be 100644
--- a/scripts/NixManifest.pm.in
+++ b/scripts/NixManifest.pm.in
@@ -219,11 +219,12 @@ sub updateManifestDB {
     # Open/create the database.
     our $dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "")
         or die "cannot open database `$dbPath'";
-    $dbh->{AutoCommit} = 0;
     $dbh->{RaiseError} = 1;
     $dbh->{PrintError} = 0;
 
     $dbh->do("pragma foreign_keys = on");
+    $dbh->do("pragma synchronous = off"); # we can always reproduce the cache
+    $dbh->do("pragma journal_mode = truncate");
 
     # Initialise the database schema, if necessary.
     $dbh->do(<<EOF);
@@ -278,6 +279,8 @@ EOF
     open MAINLOCK, ">>$manifestDir/cache.lock" or die;
     flock(MAINLOCK, LOCK_EX) or die;
 
+    $dbh->begin_work;
+
     # Read each manifest in $manifestDir and add it to the database,
     # unless we've already done so on a previous run.
     my %seen;
@@ -291,6 +294,8 @@ EOF
             "select 1 from Manifests where path = ? and timestamp = ?",
             {}, $manifest, $timestamp)} == 1;
 
+        print STDERR "caching $manifest...\n";
+        
         $dbh->do("delete from Manifests where path = ?", {}, $manifest);
                  
         $dbh->do("insert into Manifests(path, timestamp) values (?, ?)",
diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in
index a6588aa9acb7..ca4d97b51aa6 100644
--- a/scripts/download-using-manifests.pl.in
+++ b/scripts/download-using-manifests.pl.in
@@ -267,6 +267,12 @@ my @path = computeSmallestDownload $targetPath;
 die "don't know how to produce $targetPath\n" if scalar @path == 0;
 
 
+# We don't need the manifest anymore, so close it as an optimisation:
+# if we still have SQLite locks blocking other processes (we
+# shouldn't), this gets rid of them.
+$dbh->disconnect;
+
+
 # Traverse the shortest path, perform the actions described by the
 # edges.
 my $curStep = 1;