path: root/third_party/nix/src/libstore/derivations.cc

#include "libstore/derivations.hh"

#include <absl/strings/match.h>
#include <absl/strings/str_split.h>
#include <absl/strings/string_view.h>
#include <glog/logging.h>

#include "libproto/worker.pb.h"
#include "libstore/fs-accessor.hh"
#include "libstore/globals.hh"
#include "libstore/store-api.hh"
#include "libstore/worker-protocol.hh"
#include "libutil/istringstream_nocopy.hh"
#include "libutil/util.hh"

namespace nix {

// TODO(#statusor): looks like easy absl::Status conversion
void DerivationOutput::parseHashInfo(bool& recursive, Hash& hash) const {
  recursive = false;
  std::string algo = hashAlgo;

  if (std::string(algo, 0, 2) == "r:") {
    recursive = true;
    algo = std::string(algo, 2);
  }

  HashType hashType = parseHashType(algo);
  if (hashType == htUnknown) {
    throw Error(format("unknown hash algorithm '%1%'") % algo);
  }

  auto hash_ = Hash::deserialize(this->hash, hashType);
  hash = Hash::unwrap_throw(hash_);
}

nix::proto::Derivation_DerivationOutput DerivationOutput::to_proto() const {
  nix::proto::Derivation_DerivationOutput result;
  result.mutable_path()->set_path(path);
  result.set_hash_algo(hashAlgo);
  result.set_hash(hash);
  return result;
}

BasicDerivation BasicDerivation::from_proto(
    const nix::proto::Derivation* proto_derivation) {
  BasicDerivation result;
  result.platform = proto_derivation->platform();
  result.builder = proto_derivation->builder().path();

  for (auto [k, v] : proto_derivation->outputs()) {
    result.outputs.emplace(k, v);
  }

  result.inputSrcs.insert(proto_derivation->input_sources().paths().begin(),
                          proto_derivation->input_sources().paths().end());

  result.args.insert(result.args.end(), proto_derivation->args().begin(),
                     proto_derivation->args().end());

  for (auto [k, v] : proto_derivation->env()) {
    result.env.emplace(k, v);
  }

  return result;
}

nix::proto::Derivation BasicDerivation::to_proto() const {
  nix::proto::Derivation result;
  for (const auto& [key, output] : outputs) {
    result.mutable_outputs()->insert({key, output.to_proto()});
  }
  for (const auto& input_src : inputSrcs) {
    *result.mutable_input_sources()->add_paths() = input_src;
  }
  result.set_platform(platform);
  result.mutable_builder()->set_path(builder);
  for (const auto& arg : args) {
    result.add_args(arg);
  }

  for (const auto& [key, value] : env) {
    result.mutable_env()->insert({key, value});
  }

  return result;
}

Path BasicDerivation::findOutput(const std::string& id) const {
  auto i = outputs.find(id);
  if (i == outputs.end()) {
    throw Error(format("derivation has no output '%1%'") % id);
  }
  return i->second.path;
}

bool BasicDerivation::isBuiltin() const {
  return std::string(builder, 0, 8) == "builtin:";
}

Path writeDerivation(const ref<Store>& store, const Derivation& drv,
                     const std::string& name, RepairFlag repair) {
  PathSet references;
  references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
  for (auto& i : drv.inputDrvs) {
    references.insert(i.first);
  }
  /* Note that the outputs of a derivation are *not* references
     (that can be missing (of course) and should not necessarily be
     held during a garbage collection). */
  std::string suffix = name + drvExtension;
  std::string contents = drv.unparse();
  return settings.readOnlyMode
             ? store->computeStorePathForText(suffix, contents, references)
             : store->addTextToStore(suffix, contents, references, repair);
}

/* Read string `s' from stream `str'. */
static void expect(std::istream& str, const std::string& s) {
  char s2[s.size()];
  str.read(s2, s.size());
  if (std::string(s2, s.size()) != s) {
    throw FormatError(format("expected string '%1%'") % s);
  }
}

/* Read a C-style string from stream `str'. */
static std::string parseString(std::istream& str) {
  std::string res;
  expect(str, "\"");
  int c;
  while ((c = str.get()) != '"' && c != EOF) {
    if (c == '\\') {
      c = str.get();
      if (c == 'n') {
        res += '\n';
      } else if (c == 'r') {
        res += '\r';
      } else if (c == 't') {
        res += '\t';
      } else if (c == EOF) {
        throw FormatError("unexpected EOF while parsing C-style escape");
      } else {
        res += static_cast<char>(c);
      }
    } else {
      res += static_cast<char>(c);
    }
  }
  return res;
}

static Path parsePath(std::istream& str) {
  std::string s = parseString(str);
  if (s.empty() || s[0] != '/') {
    throw FormatError(format("bad path '%1%' in derivation") % s);
  }
  return s;
}

static bool endOfList(std::istream& str) {
  if (str.peek() == ',') {
    str.get();
    return false;
  }
  if (str.peek() == ']') {
    str.get();
    return true;
  }
  return false;
}

static StringSet parseStrings(std::istream& str, bool arePaths) {
  StringSet res;
  while (!endOfList(str)) {
    res.insert(arePaths ? parsePath(str) : parseString(str));
  }
  return res;
}

Derivation parseDerivation(const std::string& s) {
  Derivation drv;
  istringstream_nocopy str(s);
  expect(str, "Derive([");

  /* Parse the list of outputs. */
  while (!endOfList(str)) {
    DerivationOutput out;
    expect(str, "(");
    std::string id = parseString(str);
    expect(str, ",");
    out.path = parsePath(str);
    expect(str, ",");
    out.hashAlgo = parseString(str);
    expect(str, ",");
    out.hash = parseString(str);
    expect(str, ")");
    drv.outputs[id] = out;
  }

  /* Parse the list of input derivations. */
  expect(str, ",[");
  while (!endOfList(str)) {
    expect(str, "(");
    Path drvPath = parsePath(str);
    expect(str, ",[");
    drv.inputDrvs[drvPath] = parseStrings(str, false);
    expect(str, ")");
  }

  expect(str, ",[");
  drv.inputSrcs = parseStrings(str, true);
  expect(str, ",");
  drv.platform = parseString(str);
  expect(str, ",");
  drv.builder = parseString(str);

  /* Parse the builder arguments. */
  expect(str, ",[");
  while (!endOfList(str)) {
    drv.args.push_back(parseString(str));
  }

  /* Parse the environment variables. */
  expect(str, ",[");
  while (!endOfList(str)) {
    expect(str, "(");
    std::string name = parseString(str);
    expect(str, ",");
    std::string value = parseString(str);
    expect(str, ")");
    drv.env[name] = value;
  }

  expect(str, ")");
  return drv;
}

Derivation readDerivation(const Path& drvPath) {
  try {
    return parseDerivation(readFile(drvPath));
  } catch (FormatError& e) {
    throw Error(format("error parsing derivation '%1%': %2%") % drvPath %
                e.msg());
  }
}

Derivation Store::derivationFromPath(const Path& drvPath) {
  assertStorePath(drvPath);
  ensurePath(drvPath);
  auto accessor = getFSAccessor();
  try {
    return parseDerivation(accessor->readFile(drvPath));
  } catch (FormatError& e) {
    throw Error(format("error parsing derivation '%1%': %2%") % drvPath %
                e.msg());
  }
}

const char* findChunk(const char* begin) {
  while (*begin != 0 && *begin != '\"' && *begin != '\\' && *begin != '\n' &&
         *begin != '\r' && *begin != '\t') {
    begin++;
  }

  return begin;
}

static void printString(std::string& res, const std::string& s) {
  res += '"';

  const char* it = s.c_str();
  while (*it != 0) {
    const char* end = findChunk(it);
    std::copy(it, end, std::back_inserter(res));

    it = end;

    switch (*it) {
      case '"':
      case '\\':
        res += "\\";
        res += *it;
        break;
      case '\n':
        res += "\\n";
        break;
      case '\r':
        res += "\\r";
        break;
      case '\t':
        res += "\\t";
        break;
      default:
        continue;
    }

    it++;
  }

  res += '"';
}

template <class ForwardIterator>
static void printStrings(std::string& res, ForwardIterator i,
                         ForwardIterator j) {
  res += '[';
  bool first = true;
  for (; i != j; ++i) {
    if (first) {
      first = false;
    } else {
      res += ',';
    }
    printString(res, *i);
  }
  res += ']';
}

std::string Derivation::unparse() const {
  std::string s;
  s.reserve(65536);
  s += "Derive([";

  bool first = true;
  for (auto& i : outputs) {
    if (first) {
      first = false;
    } else {
      s += ',';
    }
    s += '(';
    printString(s, i.first);
    s += ',';
    printString(s, i.second.path);
    s += ',';
    printString(s, i.second.hashAlgo);
    s += ',';
    printString(s, i.second.hash);
    s += ')';
  }

  s += "],[";
  first = true;
  for (auto& i : inputDrvs) {
    if (first) {
      first = false;
    } else {
      s += ',';
    }
    s += '(';
    printString(s, i.first);
    s += ',';
    printStrings(s, i.second.begin(), i.second.end());
    s += ')';
  }

  s += "],";
  printStrings(s, inputSrcs.begin(), inputSrcs.end());

  s += ',';
  printString(s, platform);
  s += ',';
  printString(s, builder);
  s += ',';
  printStrings(s, args.begin(), args.end());

  s += ",[";
  first = true;
  for (auto& i : env) {
    if (first) {
      first = false;
    } else {
      s += ',';
    }
    s += '(';
    printString(s, i.first);
    s += ',';
    printString(s, i.second);
    s += ')';
  }

  s += "])";

  return s;
}

bool isDerivation(const std::string& fileName) {
  return absl::EndsWith(fileName, drvExtension);
}

bool BasicDerivation::isFixedOutput() const {
  return outputs.size() == 1 && outputs.begin()->first == "out" &&
         !outputs.begin()->second.hash.empty();
}

DrvHashes drvHashes;

/* Returns the hash of a derivation modulo fixed-output
   subderivations.  A fixed-output derivation is a derivation with one
   output (`out') for which an expected hash and hash algorithm are
   specified (using the `outputHash' and `outputHashAlgo'
   attributes).  We don't want changes to such derivations to
   propagate upwards through the dependency graph, changing output
   paths everywhere.

   For instance, if we change the url in a call to the `fetchurl'
   function, we do not want to rebuild everything depending on it
   (after all, (the hash of) the file being downloaded is unchanged).
   So the *output paths* should not change.  On the other hand, the
   *derivation paths* should change to reflect the new dependency
   graph.

   That's what this function does: it returns a hash which is just the
   hash of the derivation ATerm, except that any input derivation
   paths have been replaced by the result of a recursive call to this
   function, and that for fixed-output derivations we return a hash of
   its output path. */
Hash hashDerivationModulo(Store& store, Derivation drv) {
  /* Return a fixed hash for fixed-output derivations. */
  if (drv.isFixedOutput()) {
    auto i = drv.outputs.begin();
    return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" +
                                    i->second.hash + ":" + i->second.path);
  }

  /* For other derivations, replace the inputs paths with recursive
     calls to this function.*/
  DerivationInputs inputs2;
  for (auto& i : drv.inputDrvs) {
    Hash h = drvHashes[i.first];
    if (!h) {
      assert(store.isValidPath(i.first));
      Derivation drv2 = readDerivation(store.toRealPath(i.first));
      h = hashDerivationModulo(store, drv2);
      drvHashes[i.first] = h;
    }
    inputs2[h.to_string(Base16, false)] = i.second;
  }
  drv.inputDrvs = inputs2;

  return hashString(htSHA256, drv.unparse());
}

DrvPathWithOutputs parseDrvPathWithOutputs(absl::string_view path) {
  auto pos = path.find('!');
  if (pos == absl::string_view::npos) {
    return DrvPathWithOutputs(path, std::set<std::string>());
  }

  return DrvPathWithOutputs(
      path.substr(0, pos),
      absl::StrSplit(path.substr(pos + 1), absl::ByChar(','),
                     absl::SkipEmpty()));
}

Path makeDrvPathWithOutputs(const Path& drvPath,
                            const std::set<std::string>& outputs) {
  return outputs.empty() ? drvPath
                         : drvPath + "!" + concatStringsSep(",", outputs);
}

bool wantOutput(const std::string& output,
                const std::set<std::string>& wanted) {
  return wanted.empty() || wanted.find(output) != wanted.end();
}

PathSet BasicDerivation::outputPaths() const {
  PathSet paths;
  for (auto& i : outputs) {
    paths.insert(i.second.path);
  }
  return paths;
}

Source& readDerivation(Source& in, Store& store, BasicDerivation& drv) {
  drv.outputs.clear();
  auto nr = readNum<size_t>(in);
  for (size_t n = 0; n < nr; n++) {
    auto name = readString(in);
    DerivationOutput o;
    in >> o.path >> o.hashAlgo >> o.hash;
    store.assertStorePath(o.path);
    drv.outputs[name] = o;
  }

  drv.inputSrcs = readStorePaths<PathSet>(store, in);
  in >> drv.platform >> drv.builder;
  drv.args = readStrings<Strings>(in);

  nr = readNum<size_t>(in);
  for (size_t n = 0; n < nr; n++) {
    auto key = readString(in);
    auto value = readString(in);
    drv.env[key] = value;
  }

  return in;
}

Sink& operator<<(Sink& out, const BasicDerivation& drv) {
  out << drv.outputs.size();
  for (auto& i : drv.outputs) {
    out << i.first << i.second.path << i.second.hashAlgo << i.second.hash;
  }
  out << drv.inputSrcs << drv.platform << drv.builder << drv.args;
  out << drv.env.size();
  for (auto& i : drv.env) {
    out << i.first << i.second;
  }
  return out;
}

std::string hashPlaceholder(const std::string& outputName) {
  // FIXME: memoize?
  return "/" + hashString(htSHA256, "nix-output:" + outputName)
                   .to_string(Base32, false);
}

}  // namespace nix