#include "libstore/derivations.hh" #include #include #include #include #include "libproto/worker.pb.h" #include "libstore/fs-accessor.hh" #include "libstore/globals.hh" #include "libstore/store-api.hh" #include "libstore/worker-protocol.hh" #include "libutil/istringstream_nocopy.hh" #include "libutil/util.hh" namespace nix { // TODO(#statusor): looks like easy absl::Status conversion void DerivationOutput::parseHashInfo(bool& recursive, Hash& hash) const { recursive = false; std::string algo = hashAlgo; if (std::string(algo, 0, 2) == "r:") { recursive = true; algo = std::string(algo, 2); } HashType hashType = parseHashType(algo); if (hashType == htUnknown) { throw Error(format("unknown hash algorithm '%1%'") % algo); } auto hash_ = Hash::deserialize(this->hash, hashType); hash = Hash::unwrap_throw(hash_); } nix::proto::Derivation_DerivationOutput DerivationOutput::to_proto() const { nix::proto::Derivation_DerivationOutput result; result.mutable_path()->set_path(path); result.set_hash_algo(hashAlgo); result.set_hash(hash); return result; } BasicDerivation BasicDerivation::from_proto( const nix::proto::Derivation* proto_derivation, const nix::Store& store) { BasicDerivation result; result.platform = proto_derivation->platform(); result.builder = proto_derivation->builder().path(); store.assertStorePath(result.builder); for (auto [k, v] : proto_derivation->outputs()) { result.outputs.emplace(k, v); } result.inputSrcs.insert(proto_derivation->input_sources().paths().begin(), proto_derivation->input_sources().paths().end()); result.args.insert(result.args.end(), proto_derivation->args().begin(), proto_derivation->args().end()); for (auto [k, v] : proto_derivation->env()) { result.env.emplace(k, v); } return result; } nix::proto::Derivation BasicDerivation::to_proto() const { nix::proto::Derivation result; for (const auto& [key, output] : outputs) { result.mutable_outputs()->insert({key, output.to_proto()}); } for (const auto& input_src : inputSrcs) { result.mutable_input_sources()->add_paths(input_src); } result.set_platform(platform); result.mutable_builder()->set_path(builder); for (const auto& arg : args) { result.add_args(arg); } for (const auto& [key, value] : env) { result.mutable_env()->insert({key, value}); } return result; } Path BasicDerivation::findOutput(const std::string& id) const { auto i = outputs.find(id); if (i == outputs.end()) { throw Error(format("derivation has no output '%1%'") % id); } return i->second.path; } bool BasicDerivation::isBuiltin() const { return std::string(builder, 0, 8) == "builtin:"; } Path writeDerivation(const ref& store, const Derivation& drv, const std::string& name, RepairFlag repair) { PathSet references; references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); for (auto& i : drv.inputDrvs) { references.insert(i.first); } /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ std::string suffix = name + drvExtension; std::string contents = drv.unparse(); return settings.readOnlyMode ? store->computeStorePathForText(suffix, contents, references) : store->addTextToStore(suffix, contents, references, repair); } /* Read string `s' from stream `str'. */ static void expect(std::istream& str, const std::string& s) { char s2[s.size()]; str.read(s2, s.size()); if (std::string(s2, s.size()) != s) { throw FormatError(format("expected string '%1%'") % s); } } /* Read a C-style string from stream `str'. */ static std::string parseString(std::istream& str) { std::string res; expect(str, "\""); int c; while ((c = str.get()) != '"' && c != EOF) { if (c == '\\') { c = str.get(); if (c == 'n') { res += '\n'; } else if (c == 'r') { res += '\r'; } else if (c == 't') { res += '\t'; } else if (c == EOF) { throw FormatError("unexpected EOF while parsing C-style escape"); } else { res += static_cast(c); } } else { res += static_cast(c); } } return res; } static Path parsePath(std::istream& str) { std::string s = parseString(str); if (s.empty() || s[0] != '/') { throw FormatError(format("bad path '%1%' in derivation") % s); } return s; } static bool endOfList(std::istream& str) { if (str.peek() == ',') { str.get(); return false; } if (str.peek() == ']') { str.get(); return true; } return false; } static StringSet parseStrings(std::istream& str, bool arePaths) { StringSet res; while (!endOfList(str)) { res.insert(arePaths ? parsePath(str) : parseString(str)); } return res; } Derivation parseDerivation(const std::string& s) { Derivation drv; istringstream_nocopy str(s); expect(str, "Derive(["); /* Parse the list of outputs. */ while (!endOfList(str)) { DerivationOutput out; expect(str, "("); std::string id = parseString(str); expect(str, ","); out.path = parsePath(str); expect(str, ","); out.hashAlgo = parseString(str); expect(str, ","); out.hash = parseString(str); expect(str, ")"); drv.outputs[id] = out; } /* Parse the list of input derivations. */ expect(str, ",["); while (!endOfList(str)) { expect(str, "("); Path drvPath = parsePath(str); expect(str, ",["); drv.inputDrvs[drvPath] = parseStrings(str, false); expect(str, ")"); } expect(str, ",["); drv.inputSrcs = parseStrings(str, true); expect(str, ","); drv.platform = parseString(str); expect(str, ","); drv.builder = parseString(str); /* Parse the builder arguments. */ expect(str, ",["); while (!endOfList(str)) { drv.args.push_back(parseString(str)); } /* Parse the environment variables. */ expect(str, ",["); while (!endOfList(str)) { expect(str, "("); std::string name = parseString(str); expect(str, ","); std::string value = parseString(str); expect(str, ")"); drv.env[name] = value; } expect(str, ")"); return drv; } Derivation readDerivation(const Path& drvPath) { try { return parseDerivation(readFile(drvPath)); } catch (FormatError& e) { throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); } } Derivation Store::derivationFromPath(const Path& drvPath) { assertStorePath(drvPath); ensurePath(drvPath); auto accessor = getFSAccessor(); try { return parseDerivation(accessor->readFile(drvPath)); } catch (FormatError& e) { throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); } } const char* findChunk(const char* begin) { while (*begin != 0 && *begin != '\"' && *begin != '\\' && *begin != '\n' && *begin != '\r' && *begin != '\t') { begin++; } return begin; } static void printString(std::string& res, const std::string& s) { res += '"'; const char* it = s.c_str(); while (*it != 0) { const char* end = findChunk(it); std::copy(it, end, std::back_inserter(res)); it = end; switch (*it) { case '"': case '\\': res += "\\"; res += *it; break; case '\n': res += "\\n"; break; case '\r': res += "\\r"; break; case '\t': res += "\\t"; break; default: continue; } it++; } res += '"'; } template static void printStrings(std::string& res, ForwardIterator i, ForwardIterator j) { res += '['; bool first = true; for (; i != j; ++i) { if (first) { first = false; } else { res += ','; } printString(res, *i); } res += ']'; } std::string Derivation::unparse() const { std::string s; s.reserve(65536); s += "Derive(["; bool first = true; for (auto& i : outputs) { if (first) { first = false; } else { s += ','; } s += '('; printString(s, i.first); s += ','; printString(s, i.second.path); s += ','; printString(s, i.second.hashAlgo); s += ','; printString(s, i.second.hash); s += ')'; } s += "],["; first = true; for (auto& i : inputDrvs) { if (first) { first = false; } else { s += ','; } s += '('; printString(s, i.first); s += ','; printStrings(s, i.second.begin(), i.second.end()); s += ')'; } s += "],"; printStrings(s, inputSrcs.begin(), inputSrcs.end()); s += ','; printString(s, platform); s += ','; printString(s, builder); s += ','; printStrings(s, args.begin(), args.end()); s += ",["; first = true; for (auto& i : env) { if (first) { first = false; } else { s += ','; } s += '('; printString(s, i.first); s += ','; printString(s, i.second); s += ')'; } s += "])"; return s; } bool isDerivation(const std::string& fileName) { return absl::EndsWith(fileName, drvExtension); } bool BasicDerivation::isFixedOutput() const { return outputs.size() == 1 && outputs.begin()->first == "out" && !outputs.begin()->second.hash.empty(); } DrvHashes drvHashes; /* Returns the hash of a derivation modulo fixed-output subderivations. A fixed-output derivation is a derivation with one output (`out') for which an expected hash and hash algorithm are specified (using the `outputHash' and `outputHashAlgo' attributes). We don't want changes to such derivations to propagate upwards through the dependency graph, changing output paths everywhere. For instance, if we change the url in a call to the `fetchurl' function, we do not want to rebuild everything depending on it (after all, (the hash of) the file being downloaded is unchanged). So the *output paths* should not change. On the other hand, the *derivation paths* should change to reflect the new dependency graph. That's what this function does: it returns a hash which is just the hash of the derivation ATerm, except that any input derivation paths have been replaced by the result of a recursive call to this function, and that for fixed-output derivations we return a hash of its output path. */ Hash hashDerivationModulo(Store& store, Derivation drv) { /* Return a fixed hash for fixed-output derivations. */ if (drv.isFixedOutput()) { auto i = drv.outputs.begin(); return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" + i->second.hash + ":" + i->second.path); } /* For other derivations, replace the inputs paths with recursive calls to this function.*/ DerivationInputs inputs2; for (auto& i : drv.inputDrvs) { Hash h = drvHashes[i.first]; if (!h) { assert(store.isValidPath(i.first)); Derivation drv2 = readDerivation(store.toRealPath(i.first)); h = hashDerivationModulo(store, drv2); drvHashes[i.first] = h; } inputs2[h.to_string(Base16, false)] = i.second; } drv.inputDrvs = inputs2; return hashString(htSHA256, drv.unparse()); } DrvPathWithOutputs parseDrvPathWithOutputs(absl::string_view path) { auto pos = path.find('!'); if (pos == absl::string_view::npos) { return DrvPathWithOutputs(path, std::set()); } return DrvPathWithOutputs( path.substr(0, pos), absl::StrSplit(path.substr(pos + 1), absl::ByChar(','), absl::SkipEmpty())); } Path makeDrvPathWithOutputs(const Path& drvPath, const std::set& outputs) { return outputs.empty() ? drvPath : drvPath + "!" + concatStringsSep(",", outputs); } bool wantOutput(const std::string& output, const std::set& wanted) { return wanted.empty() || wanted.find(output) != wanted.end(); } PathSet BasicDerivation::outputPaths() const { PathSet paths; for (auto& i : outputs) { paths.insert(i.second.path); } return paths; } Source& readDerivation(Source& in, Store& store, BasicDerivation& drv) { drv.outputs.clear(); auto nr = readNum(in); for (size_t n = 0; n < nr; n++) { auto name = readString(in); DerivationOutput o; in >> o.path >> o.hashAlgo >> o.hash; store.assertStorePath(o.path); drv.outputs[name] = o; } drv.inputSrcs = readStorePaths(store, in); in >> drv.platform >> drv.builder; drv.args = readStrings(in); nr = readNum(in); for (size_t n = 0; n < nr; n++) { auto key = readString(in); auto value = readString(in); drv.env[key] = value; } return in; } Sink& operator<<(Sink& out, const BasicDerivation& drv) { out << drv.outputs.size(); for (auto& i : drv.outputs) { out << i.first << i.second.path << i.second.hashAlgo << i.second.hash; } out << drv.inputSrcs << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto& i : drv.env) { out << i.first << i.second; } return out; } std::string hashPlaceholder(const std::string& outputName) { // FIXME: memoize? return "/" + hashString(htSHA256, "nix-output:" + outputName) .to_string(Base32, false); } } // namespace nix