* Merged the SQLite branch.

13 files changed, 122 insertions, 53 deletions

diff --git a/tests/Makefile.am b/tests/Makefile.am
index beb7852db31c..fb4a2285c1d5 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -7,7 +7,8 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
   fallback.sh nix-push.sh gc.sh gc-concurrent.sh verify.sh nix-pull.sh \
   referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
   gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \
-  remote-store.sh export.sh export-graph.sh negative-caching.sh
+  remote-store.sh export.sh export-graph.sh negative-caching.sh \
+  binary-patching.sh
 
 XFAIL_TESTS =
 
@@ -31,5 +32,6 @@ EXTRA_DIST = $(TESTS) \
   filter-source.nix \
   export-graph.nix \
   negative-caching.nix \
+  binary-patching.nix \
   $(wildcard lang/*.nix) $(wildcard lang/*.exp) $(wildcard lang/*.exp.xml) $(wildcard lang/*.flags) \
   common.sh.in
diff --git a/tests/binary-patching.nix b/tests/binary-patching.nix
new file mode 100644
index 000000000000..0de3028bd64d
--- /dev/null
+++ b/tests/binary-patching.nix
@@ -0,0 +1,18 @@
+{ version }:
+
+with import ./config.nix;
+
+mkDerivation {
+  name = "foo-${toString version}";
+  builder = builtins.toFile "builder.sh"
+    ''
+      mkdir $out
+      seq 1 1000000 > $out/foo
+      ${if version != 1 then ''
+        seq 1000000 1010000 >> $out/foo
+      '' else ""}
+      ${if version == 3 then ''
+        echo foobar >> $out/foo
+      '' else ""}
+    '';
+}
diff --git a/tests/binary-patching.sh b/tests/binary-patching.sh
new file mode 100644
index 000000000000..60e57b4b00dc
--- /dev/null
+++ b/tests/binary-patching.sh
@@ -0,0 +1,58 @@
+source common.sh
+
+clearManifests
+
+mkdir -p $TEST_ROOT/cache2 $TEST_ROOT/patches
+
+RESULT=$TEST_ROOT/result
+
+# Build version 1 and 2 of the "foo" package.
+$NIX_BIN_DIR/nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest1 \
+    $($nixbuild -o $RESULT binary-patching.nix --arg version 1)
+
+out2=$($nixbuild -o $RESULT binary-patching.nix --arg version 2)
+$NIX_BIN_DIR/nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest2 $out2
+    
+out3=$($nixbuild -o $RESULT binary-patching.nix --arg version 3)
+$NIX_BIN_DIR/nix-push --copy $TEST_ROOT/cache2 $TEST_ROOT/manifest3 $out3
+
+rm $RESULT
+
+# Generate binary patches.
+$NIX_BIN_DIR/nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \
+    file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest2
+
+$NIX_BIN_DIR/nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \
+    file://$TEST_ROOT/patches $TEST_ROOT/manifest2 $TEST_ROOT/manifest3
+
+grep -q "patch {" $TEST_ROOT/manifest3
+
+# Get rid of versions 2 and 3.
+$nixstore --delete $out2 $out3
+
+# Pull the manifest containing the patches.
+clearManifests
+$NIX_BIN_DIR/nix-pull file://$TEST_ROOT/manifest3
+
+# Make sure that the download size prediction uses the patches rather
+# than the full download.
+$nixbuild -o $RESULT binary-patching.nix --arg version 3 --dry-run 2>&1 | grep -q "0.01 MiB"
+
+# Now rebuild it.  This should use the two patches generated above.
+rm -f $TEST_ROOT/var/log/nix/downloads
+$nixbuild -o $RESULT binary-patching.nix --arg version 3
+rm $RESULT
+[ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 2 ]
+
+# Add a patch from version 1 directly to version 3.
+$NIX_BIN_DIR/nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \
+    file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest3
+
+# Rebuild version 3.  This should use the direct patch rather than the
+# sequence of two patches.
+$nixstore --delete $out2 $out3
+clearManifests
+rm $TEST_ROOT/var/log/nix/downloads
+$NIX_BIN_DIR/nix-pull file://$TEST_ROOT/manifest3
+$nixbuild -o $RESULT binary-patching.nix --arg version 3
+[ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 1 ]
diff --git a/tests/build-hook.hook.sh b/tests/build-hook.hook.sh
index 83fa3bf78757..ff709985d3af 100755
--- a/tests/build-hook.hook.sh
+++ b/tests/build-hook.hook.sh
@@ -2,20 +2,22 @@
 
 #set -x
 
-drv=$4
+while read x y drv rest; do
 
-echo "HOOK for $drv" >&2
+    echo "HOOK for $drv" >&2
 
-outPath=`sed 's/Derive(\[("out",\"\([^\"]*\)\".*/\1/' $drv`
+    outPath=`sed 's/Derive(\[("out",\"\([^\"]*\)\".*/\1/' $drv`
 
-echo "output path is $outPath" >&2
+    echo "output path is $outPath" >&2
 
-if `echo $outPath | grep -q input-1`; then
-    echo "# accept" >&2
-    read x
-    echo "got $x"
-    mkdir $outPath
-    echo "BAR" > $outPath/foo
-else
-    echo "# decline" >&2
-fi
+    if `echo $outPath | grep -q input-1`; then
+        echo "# accept" >&2
+        read inputs
+        read outputs
+        mkdir $outPath
+        echo "BAR" > $outPath/foo
+    else
+        echo "# decline" >&2
+    fi
+
+done
\ No newline at end of file
diff --git a/tests/common.sh.in b/tests/common.sh.in
index b30bc44b5cc0..2c47d75eea23 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -38,6 +38,7 @@ export dot=@dot@
 export xmllint="@xmllint@"
 export xmlflags="@xmlflags@"
 export xsltproc="@xsltproc@"
+export sqlite3="@sqlite_bin@/bin/sqlite3"
 export SHELL="@shell@"
 
 export version=@version@
diff --git a/tests/init.sh b/tests/init.sh
index 0639a70662cd..104da432eaef 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -23,28 +23,33 @@ ln -s $nixinstantiate $NIX_BIN_DIR/
 ln -s $nixhash $NIX_BIN_DIR/
 ln -s $nixenv $NIX_BIN_DIR/
 ln -s $nixworker $NIX_BIN_DIR/
+ln -s $TOP/src/bsdiff-*/bsdiff $NIX_BIN_DIR/
+ln -s $TOP/src/bsdiff-*/bspatch $NIX_BIN_DIR/
 ln -s $TOP/scripts/nix-prefetch-url $NIX_BIN_DIR/
 ln -s $TOP/scripts/nix-collect-garbage $NIX_BIN_DIR/
 ln -s $TOP/scripts/nix-build $NIX_BIN_DIR/
 ln -s $TOP/scripts/nix-install-package $NIX_BIN_DIR/
 ln -s $TOP/scripts/nix-push $NIX_BIN_DIR/
 ln -s $TOP/scripts/nix-pull $NIX_BIN_DIR/
+ln -s $TOP/scripts/nix-generate-patches $NIX_BIN_DIR/
 mkdir $NIX_BIN_DIR/nix
 ln -s $bzip2_bin_test/bzip2 $NIX_BIN_DIR/nix/
 ln -s $bzip2_bin_test/bunzip2 $NIX_BIN_DIR/nix/
 ln -s $TOP/scripts/copy-from-other-stores.pl $NIX_BIN_DIR/nix/
 ln -s $TOP/scripts/download-using-manifests.pl $NIX_BIN_DIR/nix/
-ln -s $TOP/scripts/readmanifest.pm $NIX_BIN_DIR/nix/
+ln -s $TOP/scripts/GeneratePatches.pm $NIX_BIN_DIR/nix/
+ln -s $TOP/scripts/NixManifest.pm $NIX_BIN_DIR/nix/
 
 cat > "$NIX_CONF_DIR"/nix.conf <<EOF
 gc-keep-outputs = false
 gc-keep-derivations = false
 env-keep-derivations = false
+fsync-metadata = false
 EOF
 
 mkdir $NIX_DATA_DIR/nix
 cp -pr $TOP/corepkgs $NIX_DATA_DIR/nix/
-# Bah, script has the prefix hard-coded.  This is really messy stuff
+# Bah, scripts have the prefix hard-coded.  This is really messy stuff
 # (and likely to fail).
 for i in \
     $NIX_DATA_DIR/nix/corepkgs/nar/nar.sh \
@@ -56,7 +61,9 @@ for i in \
     $NIX_BIN_DIR/nix-install-package \
     $NIX_BIN_DIR/nix-push \
     $NIX_BIN_DIR/nix-pull \
-    $NIX_BIN_DIR/nix/readmanifest.pm \
+    $NIX_BIN_DIR/nix-generate-patches \
+    $NIX_BIN_DIR/nix/NixManifest.pm \
+    $NIX_BIN_DIR/nix/GeneratePatches.pm \
     ; do
     sed < $i > $i.tmp \
         -e "s^$REAL_BIN_DIR/nix-store^$NIX_BIN_DIR/nix-store^" \
@@ -96,7 +103,6 @@ mv $NIX_BIN_DIR/nix/download-using-manifests.pl $NIX_BIN_DIR/nix/substituters/do
 $nixstore --init
 
 # Did anything happen?
-test -e "$NIX_DB_DIR"/info
-test -e "$NIX_DB_DIR"/referrer
+test -e "$NIX_DB_DIR"/db.sqlite
 
 echo 'Hello World' > ./dummy
diff --git a/tests/lang/eval-okay-substring.exp b/tests/lang/eval-okay-substring.exp
index d936b7e96f94..6aace04b0f57 100644
--- a/tests/lang/eval-okay-substring.exp
+++ b/tests/lang/eval-okay-substring.exp
@@ -1 +1 @@
-"ooxfoobarybarzobaabb"
+"ooxfoobarybarzobaabbc"
diff --git a/tests/lang/eval-okay-substring.nix b/tests/lang/eval-okay-substring.nix
index 184d72580c51..424af00d9b3b 100644
--- a/tests/lang/eval-okay-substring.nix
+++ b/tests/lang/eval-okay-substring.nix
@@ -17,3 +17,5 @@ substring 1 2 s
 + substring 3 0 s
 + "b"
 + substring 3 1 s
++ "c"
++ substring 5 10 "perl"
diff --git a/tests/nix-push.sh b/tests/nix-push.sh
index a9a2f5f9c06a..0a35e3b97ab7 100644
--- a/tests/nix-push.sh
+++ b/tests/nix-push.sh
@@ -5,7 +5,7 @@ outPath=$($nixstore -r $drvPath)
 
 echo "pushing $drvPath"
 
-mkdir $TEST_ROOT/cache
+mkdir -p $TEST_ROOT/cache
 
 $NIX_BIN_DIR/nix-push \
     --copy $TEST_ROOT/cache $TEST_ROOT/manifest $drvPath
diff --git a/tests/referrers.sh b/tests/referrers.sh
index aa5edb555139..a0c195d5ab6c 100644
--- a/tests/referrers.sh
+++ b/tests/referrers.sh
@@ -1,9 +1,8 @@
 source common.sh
 
-# This takes way to long on Cygwin (because process creation is so slow...).
-if test "$system" = i686-cygwin; then exit 0; fi
+clearStore
 
-max=1000
+max=500
 
 reference=$NIX_STORE_DIR/abcdef
 touch $reference
@@ -13,46 +12,23 @@ echo "making registration..."
 
 for ((n = 0; n < $max; n++)); do
     storePath=$NIX_STORE_DIR/$n
-    touch $storePath
+    echo -n > $storePath
     ref2=$NIX_STORE_DIR/$((n+1))
     if test $((n+1)) = $max; then
         ref2=$reference
     fi
-    (echo $storePath && echo && echo 2 && echo $reference && echo $ref2)
+    echo $storePath; echo; echo 2; echo $reference; echo $ref2
 done > $TEST_ROOT/reg_info
 
 echo "registering..."
 
-time $nixstore --register-validity < $TEST_ROOT/reg_info
-
-oldTime=$(cat test-tmp/db/info/1 | grep Registered-At)
-
-echo "sleeping..."
-
-sleep 2
-
-echo "reregistering..."
-
-time $nixstore --register-validity --reregister < $TEST_ROOT/reg_info
-
-newTime=$(cat test-tmp/db/info/1 | grep Registered-At)
-
-if test "$newTime" != "$oldTime"; then
-    echo "reregistration changed original registration time"
-    exit 1
-fi
-
-if test "$(cat test-tmp/db/referrer/1 | wc -w)" -ne 1; then
-    echo "reregistration duplicated referrers"
-    exit 1
-fi
+$nixstore --register-validity < $TEST_ROOT/reg_info
 
 echo "collecting garbage..."
 ln -sfn $reference "$NIX_STATE_DIR"/gcroots/ref
-time $nixstore --gc
+$nixstore --gc
 
-if test "$(cat test-tmp/db/referrer/abcdef | wc -w)" -ne 0; then
+if test "$(sqlite3 ./test-tmp/db/db.sqlite 'select count(*) from Refs')" -ne 0; then
     echo "referrers not cleaned up"
     exit 1
 fi
-
diff --git a/tests/substituter.sh b/tests/substituter.sh
index 94ae26472501..65c4fa1856d0 100755
--- a/tests/substituter.sh
+++ b/tests/substituter.sh
@@ -16,7 +16,8 @@ if test $1 = "--query"; then
             echo 1
             echo "" # deriver
             echo 0 # nr of refs
-            echo 0 # download size
+            echo $((1 * 1024 * 1024)) # download size
+            echo $((2 * 1024 * 1024)) # nar size
         else
             echo "bad command $cmd"
             exit 1
diff --git a/tests/substituter2.sh b/tests/substituter2.sh
index 70f7d0943b98..c56a1bc47b5c 100755
--- a/tests/substituter2.sh
+++ b/tests/substituter2.sh
@@ -16,6 +16,7 @@ if test $1 = "--query"; then
             echo "" # deriver
             echo 0 # nr of refs
             echo 0 # download size
+            echo 0 # nar size
         else
             echo "bad command $cmd"
             exit 1
diff --git a/tests/substitutes.sh b/tests/substitutes.sh
index 042827314bd9..b48576c8c302 100644
--- a/tests/substitutes.sh
+++ b/tests/substitutes.sh
@@ -14,6 +14,8 @@ echo $outPath > $TEST_ROOT/sub-paths
 
 export NIX_SUBSTITUTERS=$(pwd)/substituter.sh
 
+$nixstore -r "$drvPath" --dry-run 2>&1 | grep -q "1.00 MiB.*2.00 MiB"
+
 $nixstore -rvv "$drvPath"
 
 text=$(cat "$outPath"/hello)