From 64519cfd657d024ae6e2bb74cb21ad21b886fd2a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Dec 2008 15:06:30 +0000 Subject: * Unify the treatment of sources copied to the store, and recursive SHA-256 outputs of fixed-output derivations. I.e. they now produce the same store path: $ nix-store --add x /nix/store/j2fq9qxvvxgqymvpszhs773ncci45xsj-x $ nix-store --add-fixed --recursive sha256 x /nix/store/j2fq9qxvvxgqymvpszhs773ncci45xsj-x the latter being the same as the path that a derivation derivation { name = "x"; outputHashAlgo = "sha256"; outputHashMode = "recursive"; outputHash = "..."; ... }; produces. This does change the output path for such fixed-output derivations. Fortunately they are quite rare. The most common use is fetchsvn calls with SHA-256 hashes. (There are a handful of those is Nixpkgs, mostly unstable development packages.) * Documented the computation of store paths (in store-api.cc). --- src/libstore/local-store.cc | 11 +++-- src/libstore/local-store.hh | 6 +-- src/libstore/remote-store.cc | 9 ++-- src/libstore/remote-store.hh | 6 +-- src/libstore/store-api.cc | 111 ++++++++++++++++++++++++++++++++----------- src/libstore/store-api.hh | 12 ++--- 6 files changed, 108 insertions(+), 47 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9dcd13471124..e015894b9fdc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -670,14 +670,14 @@ void LocalStore::invalidatePath(const Path & path) } -Path LocalStore::addToStore(const Path & _srcPath, bool fixed, +Path LocalStore::addToStore(const Path & _srcPath, bool recursive, string hashAlgo, PathFilter & filter) { Path srcPath(absPath(_srcPath)); debug(format("adding `%1%' to the store") % srcPath); std::pair pr = - computeStorePathForPath(srcPath, fixed, recursive, hashAlgo, filter); + computeStorePathForPath(srcPath, recursive, hashAlgo, filter); Path & dstPath(pr.first); Hash & h(pr.second); @@ -696,10 +696,13 @@ Path LocalStore::addToStore(const Path & _srcPath, bool fixed, copyPath(srcPath, dstPath, filter); + /* !!! */ +#if 0 Hash h2 = hashPath(htSHA256, dstPath, filter); if (h != h2) throw Error(format("contents of `%1%' changed while copying it to `%2%' (%3% -> %4%)") % srcPath % dstPath % printHash(h) % printHash(h2)); +#endif canonicalisePathMetaData(dstPath); @@ -713,10 +716,10 @@ Path LocalStore::addToStore(const Path & _srcPath, bool fixed, } -Path LocalStore::addTextToStore(const string & suffix, const string & s, +Path LocalStore::addTextToStore(const string & name, const string & s, const PathSet & references) { - Path dstPath = computeStorePathForText(suffix, s, references); + Path dstPath = computeStorePathForText(name, s, references); addTempRoot(dstPath); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 37ed543fc3dc..3d47446f6e40 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -89,11 +89,11 @@ public: bool querySubstitutablePathInfo(const Path & substituter, const Path & path, SubstitutablePathInfo & info); - Path addToStore(const Path & srcPath, bool fixed = false, - bool recursive = false, string hashAlgo = "", + Path addToStore(const Path & srcPath, + bool recursive = true, string hashAlgo = "sha256", PathFilter & filter = defaultPathFilter); - Path addTextToStore(const string & suffix, const string & s, + Path addTextToStore(const string & name, const string & s, const PathSet & references); void exportPath(const Path & path, bool sign, diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7e5fa2d865d9..f79b22310b5b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -278,14 +278,15 @@ Path RemoteStore::queryDeriver(const Path & path) } -Path RemoteStore::addToStore(const Path & _srcPath, bool fixed, +Path RemoteStore::addToStore(const Path & _srcPath, bool recursive, string hashAlgo, PathFilter & filter) { Path srcPath(absPath(_srcPath)); writeInt(wopAddToStore, to); writeString(baseNameOf(srcPath), to); - writeInt(fixed ? 1 : 0, to); + /* backwards compatibility hack */ + writeInt((hashAlgo == "sha256" && recursive) ? 0 : 1, to); writeInt(recursive ? 1 : 0, to); writeString(hashAlgo, to); dumpPath(srcPath, to, filter); @@ -294,11 +295,11 @@ Path RemoteStore::addToStore(const Path & _srcPath, bool fixed, } -Path RemoteStore::addTextToStore(const string & suffix, const string & s, +Path RemoteStore::addTextToStore(const string & name, const string & s, const PathSet & references) { writeInt(wopAddTextToStore, to); - writeString(suffix, to); + writeString(name, to); writeString(s, to); writeStringSet(references, to); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 969c8f9b6285..cb9124a4cdbb 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -42,11 +42,11 @@ public: bool querySubstitutablePathInfo(const Path & path, SubstitutablePathInfo & info); - Path addToStore(const Path & srcPath, bool fixed = false, - bool recursive = false, string hashAlgo = "", + Path addToStore(const Path & srcPath, + bool recursive = true, string hashAlgo = "sha256", PathFilter & filter = defaultPathFilter); - Path addTextToStore(const string & suffix, const string & s, + Path addTextToStore(const string & name, const string & s, const PathSet & references); void exportPath(const Path & path, bool sign, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ced5fd0072e8..fe4ecfad5443 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -99,55 +99,112 @@ void checkStoreName(const string & name) } +/* Store paths have the following form: + + /- + + where + + = the location of the Nix store, usually /nix/store + + = a human readable name for the path, typically obtained + from the name attribute of the derivation, or the name of the + source file from which the store path is created + + = base-32 representation of the first 160 bits of a SHA-256 + hash of ; the hash part of the store name + + = the string ":sha256:

::"; + note that it includes the location of the store as well as the + name to make sure that changes to either of those are reflected + in the hash (e.g. you won't get /nix/store/-name1 and + /nix/store/-name2 with equal hash parts). + + = one of: + "text:::..." + for plain text files written to the store using + addTextToStore(); ... are the references of the + path. + "source" + for paths copied to the store using addToStore() when recursive + = true and hashAlgo = "sha256" + "output:out" + for either the outputs created by derivations, OR paths copied + to the store using addToStore() with recursive != true or + hashAlgo != "sha256" (in that case "source" is used; it's + silly, but it's done that way for compatibility). + +

= base-16 representation of a SHA-256 hash of: + if = "text:...": + the string written to the resulting store path + if = "source": + the serialisation of the path from which this store path is + copied, as returned by hashPath() + if = "output:out": + for non-fixed derivation outputs: + the derivation (see hashDerivationModulo() in + primops.cc) + for paths copied by addToStore() or produced by fixed-output + derivations: + the string "fixed:out:::", where + = "r:" for recursive (path) hashes, or "" or flat + (file) hashes + = "md5", "sha1" or "sha256" + = base-16 representation of the path or flat hash of + the contents of the path (or expected contents of the + path for fixed-output derivations) + + It would have been nicer to handle fixed-output derivations under + "source", e.g. have something like "source:", but we're + stuck with this for now... + + The main reason for this way of computing names is to prevent name + collisions (for security). For instance, it shouldn't be feasible + to come up with a derivation whose output path collides with the + path for a copied source. The former would have a starting with + "output:out:", while the latter would have a <2> starting with + "source:". +*/ + + Path makeStorePath(const string & type, - const Hash & hash, const string & suffix) + const Hash & hash, const string & name) { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ string s = type + ":sha256:" + printHash(hash) + ":" - + nixStore + ":" + suffix; + + nixStore + ":" + name; - checkStoreName(suffix); + checkStoreName(name); return nixStore + "/" + printHash32(compressHash(hashString(htSHA256, s), 20)) - + "-" + suffix; + + "-" + name; } Path makeFixedOutputPath(bool recursive, string hashAlgo, Hash hash, string name) { - /* !!! copy/paste from primops.cc */ - Hash h = hashString(htSHA256, "fixed:out:" - + (recursive ? (string) "r:" : "") + hashAlgo + ":" - + printHash(hash) + ":" - + ""); - return makeStorePath("output:out", h, name); + return hashAlgo == "sha256" && recursive + ? makeStorePath("source", hash, name) + : makeStorePath("output:out", hashString(htSHA256, + "fixed:out:" + (recursive ? (string) "r:" : "") + hashAlgo + ":" + printHash(hash) + ":"), + name); } std::pair computeStorePathForPath(const Path & srcPath, - bool fixed, bool recursive, string hashAlgo, PathFilter & filter) + bool recursive, string hashAlgo, PathFilter & filter) { - Hash h = hashPath(htSHA256, srcPath, filter); - - string baseName = baseNameOf(srcPath); - - Path dstPath; - - if (fixed) { - HashType ht(parseHashType(hashAlgo)); - Hash h2 = recursive ? hashPath(ht, srcPath, filter) : hashFile(ht, srcPath); - dstPath = makeFixedOutputPath(recursive, hashAlgo, h2, baseName); - } - - else dstPath = makeStorePath("source", h, baseName); - + HashType ht(parseHashType(hashAlgo)); + Hash h = recursive ? hashPath(ht, srcPath, filter) : hashFile(ht, srcPath); + string name = baseNameOf(srcPath); + Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name); return std::pair(dstPath, h); } -Path computeStorePathForText(const string & suffix, const string & s, +Path computeStorePathForText(const string & name, const string & s, const PathSet & references) { Hash hash = hashString(htSHA256, s); @@ -159,7 +216,7 @@ Path computeStorePathForText(const string & suffix, const string & s, type += ":"; type += *i; } - return makeStorePath(type, hash, suffix); + return makeStorePath(type, hash, name); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ed7e60146029..adfd40a91908 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -173,13 +173,13 @@ public: derivation is pre-loaded into the Nix store. The function object `filter' can be used to exclude files (see libutil/archive.hh). */ - virtual Path addToStore(const Path & srcPath, bool fixed = false, - bool recursive = false, string hashAlgo = "", + virtual Path addToStore(const Path & srcPath, + bool recursive = true, string hashAlgo = "sha256", PathFilter & filter = defaultPathFilter) = 0; /* Like addToStore, but the contents written to the output path is a regular file containing the given string. */ - virtual Path addTextToStore(const string & suffix, const string & s, + virtual Path addTextToStore(const string & name, const string & s, const PathSet & references) = 0; /* Export a store path, that is, create a NAR dump of the store @@ -274,7 +274,7 @@ Path followLinksToStorePath(const Path & path); /* Constructs a unique store path name. */ Path makeStorePath(const string & type, - const Hash & hash, const string & suffix); + const Hash & hash, const string & name); Path makeFixedOutputPath(bool recursive, string hashAlgo, Hash hash, string name); @@ -285,7 +285,7 @@ Path makeFixedOutputPath(bool recursive, Returns the store path and the cryptographic hash of the contents of srcPath. */ std::pair computeStorePathForPath(const Path & srcPath, - bool fixed = false, bool recursive = false, string hashAlgo = "", + bool recursive = true, string hashAlgo = "sha256", PathFilter & filter = defaultPathFilter); /* Preparatory part of addTextToStore(). @@ -302,7 +302,7 @@ std::pair computeStorePathForPath(const Path & srcPath, simply yield a different store path, so other users wouldn't be affected), but it has some backwards compatibility issues (the hashing scheme changes), so I'm not doing that for now. */ -Path computeStorePathForText(const string & suffix, const string & s, +Path computeStorePathForText(const string & name, const string & s, const PathSet & references); -- cgit 1.4.1