about summary refs log tree commit diff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--doc/manual/opt-common-syn.xml1
-rw-r--r--doc/manual/opt-common.xml25
-rw-r--r--src/libmain/shared.cc2
-rw-r--r--src/libstore/db.cc7
-rw-r--r--src/libstore/db.hh3
-rw-r--r--src/libstore/globals.cc2
-rw-r--r--src/libstore/globals.hh4
-rw-r--r--src/libstore/normalise.cc135
-rw-r--r--src/libstore/normalise.hh14
-rw-r--r--src/libstore/store.cc29
-rw-r--r--src/libstore/store.hh3
-rw-r--r--src/nix-env/main.cc3
-rw-r--r--src/nix-store/main.cc6
-rw-r--r--tests/Makefile.am6
-rw-r--r--tests/fallback.nix.in (renamed from tests/fall-back.nix.in)0
-rw-r--r--tests/fallback.sh (renamed from tests/fall-back.sh)2
-rw-r--r--tests/verify.sh1
17 files changed, 185 insertions, 58 deletions
diff --git a/doc/manual/opt-common-syn.xml b/doc/manual/opt-common-syn.xml
index f900418635..f848ad2499 100644
--- a/doc/manual/opt-common-syn.xml
+++ b/doc/manual/opt-common-syn.xml
@@ -15,3 +15,4 @@
 <arg><option>-k</option></arg>
 <arg><option>--keep-failed</option></arg>
 <arg><option>-K</option></arg>
+<arg><option>--fallback</option></arg>
diff --git a/doc/manual/opt-common.xml b/doc/manual/opt-common.xml
index 158e282fdb..db8a60ecad 100644
--- a/doc/manual/opt-common.xml
+++ b/doc/manual/opt-common.xml
@@ -147,3 +147,28 @@
   </listitem>
 </varlistentry>
 
+
+<varlistentry>
+  <term><option>--fallback</option></term>
+  <listitem>
+    <para>
+      Whenever Nix attempts to realise a derivation for which a
+      closure is already known, but this closure cannot be realised,
+      fall back on normalising the derivation.
+    </para>
+
+    <para>
+      The most common scenario in which this is useful is when we have
+      registered substitutes in order to perform binary distribution
+      from, say, a network repository.  If the repository is down, the
+      realisation of the derivation will fail.  When this option is
+      specified, Nix will build the derivation instead.  Thus,
+      binary installation falls back on a source installation.  This
+      option is not the default since it is generally not desirable
+      for a transient failure in obtaining the substitutes to lead to
+      a full build from source (with the related consumption of
+      resources).
+    </para>
+  </listitem>
+</varlistentry>
+
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 13ad4fedea..6aad03a37a 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -139,6 +139,8 @@ static void initAndRun(int argc, char * * argv)
             keepFailed = true;
         else if (arg == "--keep-going" || arg == "-k")
             keepGoing = true;
+        else if (arg == "--fallback")
+            tryFallback = true;
         else if (arg == "--max-jobs" || arg == "-j") {
             ++i;
             if (i == args.end()) throw UsageError("`--max-jobs' requires an argument");
diff --git a/src/libstore/db.cc b/src/libstore/db.cc
index e792a371b8..5c8e7edecc 100644
--- a/src/libstore/db.cc
+++ b/src/libstore/db.cc
@@ -366,9 +366,12 @@ void Database::setString(const Transaction & txn, TableId table,
 
 
 void Database::setStrings(const Transaction & txn, TableId table,
-    const string & key, const Strings & data)
+    const string & key, const Strings & data, bool deleteEmpty)
 {
-    setString(txn, table, key, packStrings(data));
+    if (deleteEmpty && data.size() == 0)
+        delPair(txn, table, key);
+    else
+        setString(txn, table, key, packStrings(data));
 }
 
 
diff --git a/src/libstore/db.hh b/src/libstore/db.hh
index 1c681b9b54..bbeabfc7df 100644
--- a/src/libstore/db.hh
+++ b/src/libstore/db.hh
@@ -76,7 +76,8 @@ public:
         const string & key, const string & data);
 
     void setStrings(const Transaction & txn, TableId table,
-        const string & key, const Strings & data);
+        const string & key, const Strings & data,
+        bool deleteEmpty = true);
 
     void delPair(const Transaction & txn, TableId table,
         const string & key);
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index e7b32244b3..aad26501b7 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -10,6 +10,8 @@ bool keepFailed = false;
 
 bool keepGoing = false;
 
+bool tryFallback = false;
+
 Verbosity buildVerbosity = lvlDebug;
 
 unsigned int maxBuildJobs = 1;
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index cef4f704e6..7f88d5c53b 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -33,6 +33,10 @@ extern bool keepFailed;
    of the same goal) fails. */
 extern bool keepGoing;
 
+/* Whether, if we cannot realise the known closure corresponding to a
+   derivation, we should try to normalise the derivation instead. */
+extern bool tryFallback;
+
 /* Verbosity level for build output. */
 extern Verbosity buildVerbosity;
 
diff --git a/src/libstore/normalise.cc b/src/libstore/normalise.cc
index a38bee60f3..6fc3bdfc33 100644
--- a/src/libstore/normalise.cc
+++ b/src/libstore/normalise.cc
@@ -201,10 +201,27 @@ void Goal::waiteeDone(GoalPtr waitee, bool success)
 {
     assert(waitees.find(waitee) != waitees.end());
     waitees.erase(waitee);
-    assert(nrWaitees > 0);
+    
     if (!success) ++nrFailed;
-    if (!--nrWaitees || (!success && !keepGoing))
+    
+    assert(nrWaitees > 0);
+    if (!--nrWaitees || (!success && !keepGoing)) {
+
+        /* If we failed and keepGoing is not set, we remove all
+           remaining waitees. */
+        for (Goals::iterator i = waitees.begin(); i != waitees.end(); ++i) {
+            GoalPtr goal = *i;
+            WeakGoals waiters2;
+            for (WeakGoals::iterator j = goal->waiters.begin();
+                 j != goal->waiters.end(); ++j)
+                if (j->lock() != shared_from_this())
+                    waiters2.insert(*j);
+            goal->waiters = waiters2;
+        }
+        waitees.clear();
+        
         worker.wakeUp(shared_from_this());
+    }
 }
 
 
@@ -271,6 +288,17 @@ const char * * strings2CharPtrs(const Strings & ss)
 }
 
 
+/* Should only be called after an expression has been normalised. */
+Path queryNormalForm(const Path & nePath)
+{
+    StoreExpr ne = storeExprFromPath(nePath);
+    if (ne.type == StoreExpr::neClosure) return nePath;
+    Path nfPath;
+    if (!querySuccessor(nePath, nfPath)) abort();
+    return nfPath;
+}
+
+
 
 //////////////////////////////////////////////////////////////////////
 
@@ -472,13 +500,7 @@ void NormalisationGoal::inputNormalised()
     /* Inputs must also be realised before we can build this goal. */
     for (PathSet::iterator i = expr.derivation.inputs.begin();
          i != expr.derivation.inputs.end(); ++i)
-    {
-        Path neInput = *i, nfInput;
-        if (querySuccessor(neInput, nfInput))
-            neInput = nfInput;
-        /* Otherwise the input must be a closure. */
-        addWaitee(worker.makeRealisationGoal(neInput));
-    }
+        addWaitee(worker.makeRealisationGoal(queryNormalForm(*i)));
     
     resetWaitees(expr.derivation.inputs.size());
 
@@ -829,8 +851,8 @@ bool NormalisationGoal::prepareBuild()
          i != expr.derivation.inputs.end(); ++i)
     {
         checkInterrupt();
-        Path nePath = *i, nfPath;
-        if (!querySuccessor(nePath, nfPath)) nfPath = nePath;
+        Path nePath = *i;
+        Path nfPath = queryNormalForm(nePath);
         inputNFs.insert(nfPath);
         if (nfPath != nePath) inputSucs[nePath] = nfPath;
         /* !!! nfPath should be a root of the garbage collector while
@@ -1174,9 +1196,15 @@ string NormalisationGoal::name()
 class RealisationGoal : public Goal
 {
 private:
-    /* The path of the closure store expression. */
+    /* The path of the store expression. */
     Path nePath;
 
+    /* The normal form. */
+    Path nfPath;
+
+    /* Whether we should try to delete a broken successor mapping. */
+    bool tryFallback;
+
     /* The store expression stored at nePath. */
     StoreExpr expr;
     
@@ -1191,9 +1219,12 @@ public:
     
     /* The states. */
     void init();
+    void isNormalised();
     void haveStoreExpr();
     void elemFinished();
 
+    void fallBack(const format & error);
+    
     string name();
 };
 
@@ -1202,6 +1233,7 @@ RealisationGoal::RealisationGoal(const Path & _nePath, Worker & _worker)
     : Goal(_worker)
 {
     nePath = _nePath;
+    tryFallback = ::tryFallback;
     state = &RealisationGoal::init;
 }
 
@@ -1221,11 +1253,36 @@ void RealisationGoal::init()
 {
     trace("init");
 
-    /* The first thing to do is to make sure that the store expression
-       exists.  If it doesn't, it may be created through a
-       substitute. */
+    if (querySuccessor(nePath, nfPath)) {
+        isNormalised();
+        return;
+    }
+
+    /* First normalise the expression (which is a no-op if the
+       expression is already a closure). */
     resetWaitees(1);
-    addWaitee(worker.makeSubstitutionGoal(nePath));
+    addWaitee(worker.makeNormalisationGoal(nePath));
+
+    /* Since there is no successor right now, the normalisation goal
+       will perform an actual build.  So there is no sense in trying a
+       fallback if the realisation of the closure fails (it can't
+       really fail). */
+    tryFallback = false;
+
+    state = &RealisationGoal::isNormalised;
+}
+
+
+void RealisationGoal::isNormalised()
+{
+    trace("has been normalised");
+
+    nfPath = queryNormalForm(nePath);
+
+    /* Now make sure that the store expression exists.  If it doesn't,
+       it may be created through a substitute. */
+    resetWaitees(1);
+    addWaitee(worker.makeSubstitutionGoal(nfPath));
 
     state = &RealisationGoal::haveStoreExpr;
 }
@@ -1236,21 +1293,18 @@ void RealisationGoal::haveStoreExpr()
     trace("loading store expression");
 
     if (nrFailed != 0) {
-        printMsg(lvlError,
-            format("cannot realise missing store expression `%1%'")
-            % nePath);
-        amDone(false);
+        fallBack(format("cannot realise closure `%1%' since that file is missing") % nfPath);
         return;
     }
 
-    assert(isValidPath(nePath));
+    assert(isValidPath(nfPath));
 
     /* Get the store expression. */
-    expr = storeExprFromPath(nePath);
+    expr = storeExprFromPath(nfPath);
 
     /* If this is a normal form (i.e., a closure) we are also done. */
     if (expr.type != StoreExpr::neClosure)
-        throw Error(format("expected closure in `%1%'") % nePath);
+        throw Error(format("expected closure in `%1%'") % nfPath);
 
     /* Each path in the closure should exist, or should be creatable
        through a substitute. */
@@ -1269,12 +1323,11 @@ void RealisationGoal::elemFinished()
     trace("all closure elements present");
 
     if (nrFailed != 0) {
-        printMsg(lvlError,
+        fallBack(
             format("cannot realise closure `%1%': "
                 "%2% closure element(s) are not present "
                 "and could not be substituted")
-            % nePath % nrFailed);
-        amDone(false);
+            % nfPath % nrFailed);
         return;
     }
 
@@ -1282,6 +1335,21 @@ void RealisationGoal::elemFinished()
 }
 
 
+void RealisationGoal::fallBack(const format & error)
+{
+    if (tryFallback && nePath != nfPath) {
+        printMsg(lvlError, format("%1%; trying to normalise derivation instead")
+            % error);
+        tryFallback = false;
+        unregisterSuccessor(nePath);
+        init();
+    } else {
+        printMsg(lvlError, format("%1%; maybe `--fallback' will help") % error);
+        amDone(false);
+    }
+}
+
+
 string RealisationGoal::name()
 {
     return (format("realisation of `%1%'") % nePath).str();
@@ -1409,8 +1477,7 @@ void SubstitutionGoal::exprNormalised()
     }
 
     /* Realise the substitute store expression. */
-    if (!querySuccessor(sub.storeExpr, nfSub))
-        nfSub = sub.storeExpr;
+    nfSub = queryNormalForm(sub.storeExpr);
     addWaitee(worker.makeRealisationGoal(nfSub));
 
     resetWaitees(1);
@@ -1869,19 +1936,19 @@ Path normaliseStoreExpr(const Path & nePath)
     if (!worker.run(worker.makeNormalisationGoal(nePath)))
         throw Error(format("normalisation of store expression `%1%' failed") % nePath);
 
-    Path nfPath;
-    if (!querySuccessor(nePath, nfPath)) abort();
-    return nfPath;
+    return queryNormalForm(nePath);
 }
 
 
-void realiseClosure(const Path & nePath)
+Path realiseStoreExpr(const Path & nePath)
 {
-    startNest(nest, lvlDebug, format("realising closure `%1%'") % nePath);
+    startNest(nest, lvlDebug, format("realising `%1%'") % nePath);
 
     Worker worker;
     if (!worker.run(worker.makeRealisationGoal(nePath)))
-        throw Error(format("realisation of closure `%1%' failed") % nePath);
+        throw Error(format("realisation of store expressions `%1%' failed") % nePath);
+
+    return queryNormalForm(nePath);
 }
 
 
diff --git a/src/libstore/normalise.hh b/src/libstore/normalise.hh
index bbde545c40..43be136e5b 100644
--- a/src/libstore/normalise.hh
+++ b/src/libstore/normalise.hh
@@ -10,13 +10,13 @@
    successor is known. */
 Path normaliseStoreExpr(const Path & nePath);
 
-/* Realise a closure store expression in the file system. 
-
-   The pending paths are those that are already being realised.  This
-   prevents infinite recursion for paths realised through a substitute
-   (since when we build the substitute, we would first try to realise
-   its output paths through substitutes... kaboom!). */
-void realiseClosure(const Path & nePath);
+/* Realise a store expression.  If the expression is a derivation, it
+   is first normalised into a closure.  The closure is then realised
+   in the file system (i.e., it is ensured that each path in the
+   closure exists in the file system, if necessary by using the
+   substitute mechanism).  Returns the normal form of the expression
+   (i.e., its closure expression). */
+Path realiseStoreExpr(const Path & nePath);
 
 /* Ensure that a path exists, possibly by instantiating it by
    realising a substitute. */
diff --git a/src/libstore/store.cc b/src/libstore/store.cc
index 9677f84223..44b3a29e34 100644
--- a/src/libstore/store.cc
+++ b/src/libstore/store.cc
@@ -237,6 +237,30 @@ void registerSuccessor(const Transaction & txn,
 }
 
 
+void unregisterSuccessor(const Path & srcPath)
+{
+    assertStorePath(srcPath);
+
+    Transaction txn(nixDB);
+
+    Path sucPath;
+    if (!nixDB.queryString(txn, dbSuccessors, srcPath, sucPath)) {
+        txn.abort();
+        return;
+    }
+    nixDB.delPair(txn, dbSuccessors, srcPath);
+
+    Paths revs;
+    nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs);
+    Paths::iterator i = find(revs.begin(), revs.end(), srcPath);
+    assert(i != revs.end());
+    revs.erase(i);
+    nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs);
+
+    txn.commit();
+}
+
+
 bool querySuccessor(const Path & srcPath, Path & sucPath)
 {
     return nixDB.queryString(noTxn, dbSuccessors, srcPath, sucPath);
@@ -294,10 +318,7 @@ static void writeSubstitutes(const Transaction & txn,
         ss.push_back(packStrings(ss2));
     }
 
-    if (ss.size() == 0)
-        nixDB.delPair(txn, dbSubstitutes, srcPath);
-    else
-        nixDB.setStrings(txn, dbSubstitutes, srcPath, ss);
+    nixDB.setStrings(txn, dbSubstitutes, srcPath, ss);
 }
 
 
diff --git a/src/libstore/store.hh b/src/libstore/store.hh
index 40d1859e53..68f7d61905 100644
--- a/src/libstore/store.hh
+++ b/src/libstore/store.hh
@@ -54,6 +54,9 @@ void copyPath(const Path & src, const Path & dst);
 void registerSuccessor(const Transaction & txn,
     const Path & srcPath, const Path & sucPath);
 
+/* Remove a successor mapping. */
+void unregisterSuccessor(const Path & srcPath);
+
 /* Return the predecessors of the Nix expression stored at the given
    path. */
 bool querySuccessor(const Path & srcPath, Path & sucPath);
diff --git a/src/nix-env/main.cc b/src/nix-env/main.cc
index 50ff0b19e9..c57f03cce0 100644
--- a/src/nix-env/main.cc
+++ b/src/nix-env/main.cc
@@ -213,8 +213,7 @@ void createUserEnv(EvalState & state, const DrvInfos & drvs,
     
     /* Realise the resulting store expression. */
     debug(format("realising user environment"));
-    Path nfPath = normaliseStoreExpr(topLevelDrv.drvPath);
-    realiseClosure(nfPath);
+    Path nfPath = realiseStoreExpr(topLevelDrv.drvPath);
 
     /* Switch the current user environment to the output path. */
     debug(format("switching to new user environment"));
diff --git a/src/nix-store/main.cc b/src/nix-store/main.cc
index 2f38edf77b..e83f9133f6 100644
--- a/src/nix-store/main.cc
+++ b/src/nix-store/main.cc
@@ -26,8 +26,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
     for (Strings::iterator i = opArgs.begin();
          i != opArgs.end(); i++)
     {
-        Path nfPath = normaliseStoreExpr(*i);
-        realiseClosure(nfPath);
+        Path nfPath = realiseStoreExpr(*i);
         cout << format("%1%\n") % (string) nfPath;
     }
 }
@@ -58,8 +57,7 @@ static void opAdd(Strings opFlags, Strings opArgs)
 Path maybeNormalise(const Path & ne, bool normalise, bool realise)
 {
     if (realise) {
-        Path ne2 = normaliseStoreExpr(ne);
-        realiseClosure(ne2);
+        Path ne2 = realiseStoreExpr(ne);
         return normalise ? ne2 : ne;
     } else
         return normalise ? normaliseStoreExpr(ne) : ne;
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 19122cd955..9e1d6a1983 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -19,10 +19,10 @@ parallel.sh: parallel.nix
 build-hook.sh: build-hook.nix
 substitutes.sh: substitutes.nix substituter.nix
 substitutes2.sh: substitutes2.nix substituter.nix substituter2.nix
-fall-back.sh: fall-back.nix
+fallback.sh: fallback.nix
 
 TESTS = init.sh simple.sh dependencies.sh locking.sh parallel.sh \
-  build-hook.sh substitutes.sh substitutes2.sh
+  build-hook.sh substitutes.sh substitutes2.sh fallback.sh verify.sh
 
 XFAIL_TESTS =
 
@@ -36,4 +36,4 @@ EXTRA_DIST = $(TESTS) \
   build-hook.nix.in build-hook.hook.sh \
   substitutes.nix.in substituter.nix.in substituter.builder.sh \
   substitutes2.nix.in substituter2.nix.in substituter2.builder.sh \
-  fall-back.nix.in
+  fallback.nix.in
diff --git a/tests/fall-back.nix.in b/tests/fallback.nix.in
index 0575d6940d..0575d6940d 100644
--- a/tests/fall-back.nix.in
+++ b/tests/fallback.nix.in
diff --git a/tests/fall-back.sh b/tests/fallback.sh
index e4a3942177..5799775eb2 100644
--- a/tests/fall-back.sh
+++ b/tests/fallback.sh
@@ -7,7 +7,7 @@ suc=$NIX_STORE_DIR/deadbeafdeadbeafdeadbeafdeadbeaf-s.store
 (echo $suc && echo $NIX_STORE_DIR/ffffffffffffffffffffffffffffffff.store && echo "/bla" && echo 0) | $TOP/src/nix-store/nix-store --substitute
 $TOP/src/nix-store/nix-store --successor $storeExpr $suc
 
-outPath=$($TOP/src/nix-store/nix-store -qnf "$storeExpr")
+outPath=$($TOP/src/nix-store/nix-store -qnf --fallback "$storeExpr")
 
 echo "output path is $outPath"
 
diff --git a/tests/verify.sh b/tests/verify.sh
new file mode 100644
index 0000000000..ede3e7d74b
--- /dev/null
+++ b/tests/verify.sh
@@ -0,0 +1 @@
+$TOP/src/nix-store/nix-store --verify
generated by cgit-pink 1.4.1 (git 2.44.2) at 2024-07-14T17ยท13+0000