* NAR archives: handle files larger than 2^32 bytes. Previously it

  would just silently store only (fileSize % 2^32) bytes.
* Use posix_fallocate if available when unpacking archives.
* Provide a better error message when trying to unpack something that
  isn't a NAR archive.

6 files changed, 47 insertions, 17 deletions

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 1d0c68cb88a5..fdfc8534662a 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -865,7 +865,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
        store path follows the archive data proper), and besides, we
        don't know yet whether the signature is valid. */
     Path tmpDir = createTempDir(nixStore);
-    AutoDelete delTmp(tmpDir);
+    AutoDelete delTmp(tmpDir); /* !!! could be GC'ed! */
     Path unpacked = tmpDir + "/unpacked";
 
     restorePath(unpacked, hashAndReadSource);
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 30b94951bd40..3142aa929b3e 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -2,6 +2,7 @@
 #include <algorithm>
 #include <vector>
 
+#define _XOPEN_SOURCE 600
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
@@ -11,6 +12,8 @@
 #include "archive.hh"
 #include "util.hh"
 
+#include "config.h"
+
 
 namespace nix {
 
@@ -47,17 +50,17 @@ static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter)
 }
 
 
-static void dumpContents(const Path & path, unsigned int size, 
+static void dumpContents(const Path & path, off_t size, 
     Sink & sink)
 {
     writeString("contents", sink);
-    writeInt(size, sink);
+    writeLongLong(size, sink);
 
     AutoCloseFD fd = open(path.c_str(), O_RDONLY);
     if (fd == -1) throw SysError(format("opening file `%1%'") % path);
     
     unsigned char buf[65536];
-    unsigned int left = size;
+    off_t left = size;
 
     while (left > 0) {
         size_t n = left > sizeof(buf) ? sizeof(buf) : left;
@@ -101,7 +104,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
         writeString(readLink(path), sink);
     }
 
-    else throw Error("unknown file type: " + path);
+    else throw Error(format("file `%1%' has an unknown type") % path);
 
     writeString(")", sink);
 }
@@ -114,9 +117,9 @@ void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
 }
 
 
-static Error badArchive(string s)
+static SerialisationError badArchive(string s)
 {
-    return Error("bad archive: " + s);
+    return SerialisationError("bad archive: " + s);
 }
 
 
@@ -162,14 +165,17 @@ static void parseEntry(ParseSink & sink, Source & source, const Path & path)
 
 static void parseContents(ParseSink & sink, Source & source, const Path & path)
 {
-    unsigned int size = readInt(source);
-    unsigned int left = size;
+    unsigned long long size = readLongLong(source);
+    
+    sink.preallocateContents(size);
+
+    unsigned long long left = size;
     unsigned char buf[65536];
 
     while (left) {
         checkInterrupt();
         unsigned int n = sizeof(buf);
-        if (n > left) n = left;
+        if ((unsigned long long) n > left) n = left;
         source(buf, n);
         sink.receiveContents(buf, n);
         left -= n;
@@ -248,8 +254,15 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
 
 void parseDump(ParseSink & sink, Source & source)
 {
-    if (readString(source) != archiveVersion1)
-        throw badArchive("expected Nix archive");
+    string version;    
+    try {
+        version = readString(source);
+    } catch (SerialisationError & e) {
+        /* This generally means the integer at the start couldn't be
+           decoded.  Ignore and throw the exception below. */
+    }
+    if (version != archiveVersion1)
+        throw badArchive("input doesn't look like a Nix archive");
     parse(sink, source, "");
 }
 
@@ -282,6 +295,16 @@ struct RestoreSink : ParseSink
             throw SysError("fchmod");
     }
 
+    void preallocateContents(unsigned long long len)
+    {
+#if HAVE_POSIX_FALLOCATE
+        if (len) {
+            errno = posix_fallocate(fd, 0, len);
+            if (errno) throw SysError(format("preallocating file of %1% bytes") % len);
+        }
+#endif
+    }
+
     void receiveContents(unsigned char * data, unsigned int len)
     {
         writeFull(fd, data, len);
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index 06f09cd1ec45..fff62031397c 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -62,6 +62,7 @@ struct ParseSink
     
     virtual void createRegularFile(const Path & path) { };
     virtual void isExecutable() { };
+    virtual void preallocateContents(unsigned long long size) { };
     virtual void receiveContents(unsigned char * data, unsigned int len) { };
 
     virtual void createSymlink(const Path & path, const string & target) { };
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index c13e8c7e38e5..9b422271323f 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -80,7 +80,7 @@ void readPadding(unsigned int len, Source & source)
         unsigned int n = 8 - (len % 8);
         source(zero, n);
         for (unsigned int i = 0; i < n; i++)
-            if (zero[i]) throw Error("non-zero padding");
+            if (zero[i]) throw SerialisationError("non-zero padding");
     }
 }
 
@@ -90,7 +90,7 @@ unsigned int readInt(Source & source)
     unsigned char buf[8];
     source(buf, sizeof(buf));
     if (buf[4] || buf[5] || buf[6] || buf[7])
-        throw Error("implementation cannot deal with > 32-bit integers");
+        throw SerialisationError("implementation cannot deal with > 32-bit integers");
     return
         buf[0] |
         (buf[1] << 8) |
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 6a74c4cf6ea1..0e797d63bca9 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -106,6 +106,9 @@ string readString(Source & source);
 StringSet readStringSet(Source & source);
 
 
+MakeError(SerialisationError, Error)
+
+
 }
 
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index f6271d56aafd..4a70d870a563 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -617,16 +617,19 @@ static void opExport(Strings opFlags, Strings opArgs)
 static void opImport(Strings opFlags, Strings opArgs)
 {
     bool requireSignature = false;
-    for (Strings::iterator i = opFlags.begin();
-         i != opFlags.end(); ++i)
+    foreach (Strings::iterator, i, opFlags)
         if (*i == "--require-signature") requireSignature = true;
         else throw UsageError(format("unknown flag `%1%'") % *i);
     
     if (!opArgs.empty()) throw UsageError("no arguments expected");
     
     FdSource source(STDIN_FILENO);
-    while (readInt(source) == 1)
+    while (true) {
+        int n = readInt(source);
+        if (n == 0) break;
+        if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'");
         cout << format("%1%\n") % store->importPath(requireSignature, source) << std::flush;
+    }
 }