Add function queryPathFromHashPart()

To implement binary caches efficiently, Hydra needs to be able to map
the hash part of a store path (e.g. "gbg...zr7") to the full store
path (e.g. "/nix/store/gbg...kzr7-subversion-1.7.5").  (The binary
cache mechanism uses hash parts as a key for looking up store paths to
ensure privacy.)  However, doing a search in the Nix store for
/nix/store/<hash>* is expensive since it requires reading the entire
directory.  queryPathFromHashPart() prevents this by doing a cheap
database lookup.

9 files changed, 67 insertions, 0 deletions

diff --git a/perl/lib/Nix/Store.pm b/perl/lib/Nix/Store.pm
index 8312a732cdea..2e79c74fe290 100644
--- a/perl/lib/Nix/Store.pm
+++ b/perl/lib/Nix/Store.pm
@@ -14,6 +14,7 @@ our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } );
 
 our @EXPORT = qw(
     isValidPath queryReferences queryPathInfo queryDeriver queryPathHash
+    queryPathFromHashPart
     topoSortPaths computeFSClosure followLinksToStorePath exportPaths
     hashPath hashFile hashString
     addToStore makeFixedOutputPath
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 2ebff5575616..76de674e6d5b 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -108,6 +108,17 @@ SV * queryPathInfo(char * path, int base32)
         }
 
 
+SV * queryPathFromHashPart(char * hashPart)
+    PPCODE:
+        try {
+            doInit();
+            Path path = store->queryPathFromHashPart(hashPart);
+            XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
+        } catch (Error & e) {
+            croak(e.what());
+        }
+
+
 SV * computeFSClosure(int flipDirection, int includeOutputs, ...)
     PPCODE:
         try {
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 1ce62aeafcef..30398a244607 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -405,6 +405,10 @@ void LocalStore::openDB(bool create)
         "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
     stmtQueryDerivationOutputs.create(db,
         "select id, path from DerivationOutputs where drv = ?;");
+    // Use "path >= ?" with limit 1 rather than "path like '?%'" to
+    // ensure efficient lookup.
+    stmtQueryPathFromHashPart.create(db,
+        "select path from ValidPaths where path >= ? limit 1;");
 }
 
 
@@ -865,6 +869,26 @@ StringSet LocalStore::queryDerivationOutputNames(const Path & path)
 }
 
 
+Path LocalStore::queryPathFromHashPart(const string & hashPart)
+{
+    if (hashPart.size() != 32) throw Error("invalid hash part");
+    
+    SQLiteTxn txn(db);
+
+    Path prefix = nixStore + "/" + hashPart;
+    
+    SQLiteStmtUse use(stmtQueryPathFromHashPart);
+    stmtQueryPathFromHashPart.bind(prefix);
+
+    int res = sqlite3_step(stmtQueryPathFromHashPart);
+    if (res == SQLITE_DONE) return "";
+    if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database");
+
+    const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0);
+    return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";
+}
+
+
 void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run)
 {
     if (run.pid != -1) return;
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index aa8e8582fb0d..65ee029c261e 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -121,6 +121,8 @@ public:
 
     StringSet queryDerivationOutputNames(const Path & path);
     
+    Path queryPathFromHashPart(const string & hashPart);
+    
     PathSet querySubstitutablePaths();
     
     bool hasSubstitutes(const Path & path);
@@ -217,6 +219,7 @@ private:
     SQLiteStmt stmtAddDerivationOutput;
     SQLiteStmt stmtQueryValidDerivers;
     SQLiteStmt stmtQueryDerivationOutputs;
+    SQLiteStmt stmtQueryPathFromHashPart;
 
     int getSchema();
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 5e5561a6aecf..cbb70b2fd726 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -341,6 +341,18 @@ PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
 }
 
 
+Path RemoteStore::queryPathFromHashPart(const string & hashPart)
+{
+    openConnection();
+    writeInt(wopQueryPathFromHashPart, to);
+    writeString(hashPart, to);
+    processStderr();
+    Path path = readString(from);
+    if (!path.empty()) assertStorePath(path);
+    return path;
+}
+
+
 Path RemoteStore::addToStore(const Path & _srcPath,
     bool recursive, HashType hashAlgo, PathFilter & filter)
 {
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index e9f40da6dbf4..f0e5dbf7695a 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -43,6 +43,8 @@ public:
     
     StringSet queryDerivationOutputNames(const Path & path);
 
+    Path queryPathFromHashPart(const string & hashPart);
+    
     bool hasSubstitutes(const Path & path);
     
     bool querySubstitutablePathInfo(const Path & path,
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index bf3269f57818..0ab15c38063e 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -139,6 +139,10 @@ public:
 
     /* Query the output names of the derivation denoted by `path'. */
     virtual StringSet queryDerivationOutputNames(const Path & path) = 0;
+
+    /* Query the full store path given the hash part of a valid store
+       path, or "" if the path doesn't exist. */
+    virtual Path queryPathFromHashPart(const string & hashPart) = 0;
     
     /* Query whether a path has substitutes. */
     virtual bool hasSubstitutes(const Path & path) = 0;
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 6a5f0ed40d46..b08410fa1c42 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -40,6 +40,7 @@ typedef enum {
     wopQueryPathInfo = 26,
     wopImportPaths = 27,
     wopQueryDerivationOutputNames = 28,
+    wopQueryPathFromHashPart = 29,
 } WorkerOp;
 
 
diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc
index 2f0a2ab209c5..74a619c71d0a 100644
--- a/src/nix-worker/nix-worker.cc
+++ b/src/nix-worker/nix-worker.cc
@@ -350,6 +350,15 @@ static void performOp(unsigned int clientVersion,
         break;
     }
 
+    case wopQueryPathFromHashPart: {
+        string hashPart = readString(from);
+        startWork();
+        Path path = store->queryPathFromHashPart(hashPart);
+        stopWork();
+        writeString(path, to);
+        break;
+    }
+
     case wopAddToStore: {
         string baseName = readString(from);
         bool fixed = readInt(from) == 1; /* obsolete */