diff options
Diffstat (limited to 'third_party/nix/src/libexpr')
39 files changed, 9350 insertions, 0 deletions
diff --git a/third_party/nix/src/libexpr/CMakeLists.txt b/third_party/nix/src/libexpr/CMakeLists.txt new file mode 100644 index 0000000000..8cb7143d2c --- /dev/null +++ b/third_party/nix/src/libexpr/CMakeLists.txt @@ -0,0 +1,85 @@ +# -*- mode: cmake; -*- +add_library(nixexpr SHARED) +set_property(TARGET nixexpr PROPERTY CXX_STANDARD 17) +include_directories(${PROJECT_BINARY_DIR}) # for 'generated/' +target_include_directories(nixexpr PUBLIC "${nix_SOURCE_DIR}/src") + +# Generate lexer & parser for inclusion: +find_package(BISON) +find_package(FLEX) + +BISON_TARGET(NixParser parser.y + ${PROJECT_BINARY_DIR}/generated/parser-tab.cc + DEFINES_FILE ${PROJECT_BINARY_DIR}/generated/parser-tab.hh) + +FLEX_TARGET(NixLexer lexer.l + ${PROJECT_BINARY_DIR}/generated/lexer-tab.cc + DEFINES_FILE ${PROJECT_BINARY_DIR}/generated/lexer-tab.hh) + +ADD_FLEX_BISON_DEPENDENCY(NixLexer NixParser) + +set(HEADER_FILES + attr-path.hh + attr-set.hh + common-eval-args.hh + eval.hh + eval-inline.hh + function-trace.hh + get-drvs.hh + json-to-value.hh + names.hh + nixexpr.hh + parser.hh + primops.hh + symbol-table.hh + value.hh + value-to-json.hh + value-to-xml.hh +) + +target_sources(nixexpr + PUBLIC + ${HEADER_FILES} + + PRIVATE + ${PROJECT_BINARY_DIR}/generated/parser-tab.hh + ${PROJECT_BINARY_DIR}/generated/parser-tab.cc + ${PROJECT_BINARY_DIR}/generated/lexer-tab.hh + ${PROJECT_BINARY_DIR}/generated/lexer-tab.cc + primops/context.cc + primops/fetchGit.cc + primops/fetchMercurial.cc + primops/fromTOML.cc + attr-path.cc + attr-set.cc + common-eval-args.cc + eval.cc + function-trace.cc + get-drvs.cc + json-to-value.cc + names.cc + nixexpr.cc + parser.cc + primops.cc + symbol-table.cc + value.cc + value-to-json.cc + value-to-xml.cc +) + +target_link_libraries(nixexpr + nixmain + nixstore + nixutil + + absl::btree + absl::flat_hash_set + absl::node_hash_set + absl::strings +) + +configure_file("nix-expr.pc.in" "${PROJECT_BINARY_DIR}/nix-expr.pc" @ONLY) +INSTALL(FILES "${PROJECT_BINARY_DIR}/nix-expr.pc" DESTINATION "${PKGCONFIG_INSTALL_DIR}") + +INSTALL(FILES ${HEADER_FILES} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/nix/libexpr) +INSTALL(TARGETS nixexpr DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/third_party/nix/src/libexpr/attr-path.cc b/third_party/nix/src/libexpr/attr-path.cc new file mode 100644 index 0000000000..86ebeec2fb --- /dev/null +++ b/third_party/nix/src/libexpr/attr-path.cc @@ -0,0 +1,109 @@ +#include "libexpr/attr-path.hh" + +#include <absl/strings/numbers.h> + +#include "libexpr/eval-inline.hh" +#include "libutil/util.hh" + +namespace nix { + +static Strings parseAttrPath(const std::string& s) { + Strings res; + std::string cur; + std::string::const_iterator i = s.begin(); + while (i != s.end()) { + if (*i == '.') { + res.push_back(cur); + cur.clear(); + } else if (*i == '"') { + ++i; + while (true) { + if (i == s.end()) { + throw Error(format("missing closing quote in selection path '%1%'") % + s); + } + if (*i == '"') { + break; + } + cur.push_back(*i++); + } + } else { + cur.push_back(*i); + } + ++i; + } + if (!cur.empty()) { + res.push_back(cur); + } + return res; +} + +Value* findAlongAttrPath(EvalState& state, const std::string& attrPath, + Bindings* autoArgs, Value& vIn) { + Strings tokens = parseAttrPath(attrPath); + + Error attrError = + Error(format("attribute selection path '%1%' does not match expression") % + attrPath); + + Value* v = &vIn; + + for (auto& attr : tokens) { + /* Is i an index (integer) or a normal attribute name? */ + enum { apAttr, apIndex } apType = apAttr; + unsigned int attrIndex; + if (absl::SimpleAtoi(attr, &attrIndex)) { + apType = apIndex; + } + + /* Evaluate the expression. */ + Value* vNew = state.allocValue(); + state.autoCallFunction(autoArgs, *v, *vNew); + v = vNew; + state.forceValue(*v); + + /* It should evaluate to either a set or an expression, + according to what is specified in the attrPath. */ + + if (apType == apAttr) { + if (v->type != tAttrs) { + throw TypeError(format("the expression selected by the selection path " + "'%1%' should be a set but is %2%") % + attrPath % showType(*v)); + } + + if (attr.empty()) { + throw Error(format("empty attribute name in selection path '%1%'") % + attrPath); + } + + Bindings::iterator a = v->attrs->find(state.symbols.Create(attr)); + if (a == v->attrs->end()) { + throw Error( + format("attribute '%1%' in selection path '%2%' not found") % attr % + attrPath); + } + v = &*(a->second).value; + } + + else if (apType == apIndex) { + if (!v->isList()) { + throw TypeError(format("the expression selected by the selection path " + "'%1%' should be a list but is %2%") % + attrPath % showType(*v)); + } + + if (attrIndex >= v->listSize()) { + throw Error( + format("list index %1% in selection path '%2%' is out of range") % + attrIndex % attrPath); + } + + v = (*v->list)[attrIndex]; + } + } + + return v; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/attr-path.hh b/third_party/nix/src/libexpr/attr-path.hh new file mode 100644 index 0000000000..97170be840 --- /dev/null +++ b/third_party/nix/src/libexpr/attr-path.hh @@ -0,0 +1,13 @@ +#pragma once + +#include <map> +#include <string> + +#include "libexpr/eval.hh" + +namespace nix { + +Value* findAlongAttrPath(EvalState& state, const std::string& attrPath, + Bindings* autoArgs, Value& vIn); + +} diff --git a/third_party/nix/src/libexpr/attr-set.cc b/third_party/nix/src/libexpr/attr-set.cc new file mode 100644 index 0000000000..b1617c981f --- /dev/null +++ b/third_party/nix/src/libexpr/attr-set.cc @@ -0,0 +1,111 @@ +#include "libexpr/attr-set.hh" + +#include <new> + +#include <absl/container/btree_map.h> +#include <glog/logging.h> + +#include "libexpr/eval-inline.hh" + +namespace nix { + +// This function inherits its name from previous implementations, in +// which Bindings was backed by an array of elements which was scanned +// linearly. +// +// In that setup, inserting duplicate elements would always yield the +// first element (until the next sort, which wasn't stable, after +// which things are more or less undefined). +// +// This behaviour is mimicked by using .insert(), which will *not* +// override existing values. +void Bindings::push_back(const Attr& attr) { + auto [_, inserted] = attributes_.insert({attr.name, attr}); + + if (!inserted) { + DLOG(WARNING) << "attempted to insert duplicate attribute for key '" + << attr.name << "'"; + } +} + +size_t Bindings::size() const { return attributes_.size(); } + +bool Bindings::empty() { return attributes_.empty(); } + +Bindings::iterator Bindings::find(const Symbol& name) { + return attributes_.find(name); +} + +bool Bindings::Equal(const Bindings* other, EvalState& state) const { + if (this == other) { + return true; + } + + if (this->attributes_.size() != other->attributes_.size()) { + return false; + } + + Bindings::const_iterator i; + Bindings::const_iterator j; + for (i = this->cbegin(), j = other->cbegin(); i != this->cend(); ++i, ++j) { + if (i->second.name != j->second.name || + !state.eqValues(*i->second.value, *j->second.value)) { + return false; + } + } + + return true; +} + +Bindings::iterator Bindings::begin() { return attributes_.begin(); } +Bindings::iterator Bindings::end() { return attributes_.end(); } + +Bindings::const_iterator Bindings::cbegin() const { + return attributes_.cbegin(); +} + +Bindings::const_iterator Bindings::cend() const { return attributes_.cend(); } + +std::unique_ptr<Bindings> Bindings::New(size_t capacity) { + if (capacity == 0) { + // TODO(tazjin): A lot of 0-capacity Bindings are allocated. + // It would be nice to optimize that. + } + + return std::make_unique<Bindings>(); +} + +std::unique_ptr<Bindings> Bindings::Merge(const Bindings& lhs, + const Bindings& rhs) { + auto bindings = New(lhs.size() + rhs.size()); + + // Values are merged by inserting the entire iterator range of both + // input sets. The right-hand set (the values of which take + // precedence) is inserted *first* because the range insertion + // method does not override values. + bindings->attributes_.insert(rhs.attributes_.cbegin(), + rhs.attributes_.cend()); + bindings->attributes_.insert(lhs.attributes_.cbegin(), + lhs.attributes_.cend()); + + return bindings; +} + +void EvalState::mkAttrs(Value& v, size_t capacity) { + clearValue(v); + v.type = tAttrs; + v.attrs = Bindings::New(capacity); + nrAttrsets++; + nrAttrsInAttrsets += capacity; +} + +/* Create a new attribute named 'name' on an existing attribute set stored + in 'vAttrs' and return the newly allocated Value which is associated with + this attribute. */ +Value* EvalState::allocAttr(Value& vAttrs, const Symbol& name) { + Value* v = allocValue(); + vAttrs.attrs->push_back(Attr(name, v)); + return v; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/attr-set.hh b/third_party/nix/src/libexpr/attr-set.hh new file mode 100644 index 0000000000..5d77e0907c --- /dev/null +++ b/third_party/nix/src/libexpr/attr-set.hh @@ -0,0 +1,69 @@ +// This file implements the underlying structure of Nix attribute sets. +#pragma once + +#include <absl/container/btree_map.h> + +#include "libexpr/nixexpr.hh" +#include "libexpr/symbol-table.hh" +#include "libutil/types.hh" + +namespace nix { // TODO(tazjin): ::expr + +class EvalState; +struct Value; + +/* Map one attribute name to its value. */ +struct Attr { + Symbol name; + Value* value; // TODO(tazjin): Who owns this? + Pos* pos; // TODO(tazjin): Who owns this? + Attr(Symbol name, Value* value, Pos* pos = &noPos) + : name(name), value(value), pos(pos){}; +}; + +using AttributeMap = absl::btree_map<Symbol, Attr>; + +class Bindings { + public: + using iterator = AttributeMap::iterator; + using const_iterator = AttributeMap::const_iterator; + + // Allocate a new attribute set that is visible to the garbage + // collector. + static std::unique_ptr<Bindings> New(size_t capacity = 0); + + // Create a new attribute set by merging two others. This is used to + // implement the `//` operator in Nix. + static std::unique_ptr<Bindings> Merge(const Bindings& lhs, + const Bindings& rhs); + + // Return the number of contained elements. + size_t size() const; + + // Is this attribute set empty? + bool empty(); + + // Insert, but do not replace, values in the attribute set. + void push_back(const Attr& attr); + + // Are these two attribute sets deeply equal? + // Note: Does not special-case derivations. Use state.eqValues() to check + // attrsets that may be derivations. + bool Equal(const Bindings* other, EvalState& state) const; + + // Look up a specific element of the attribute set. + iterator find(const Symbol& name); + + iterator begin(); + const_iterator cbegin() const; + iterator end(); + const_iterator cend() const; + + // oh no + friend class EvalState; + + private: + AttributeMap attributes_; +}; + +} // namespace nix diff --git a/third_party/nix/src/libexpr/common-eval-args.cc b/third_party/nix/src/libexpr/common-eval-args.cc new file mode 100644 index 0000000000..f63d3f8276 --- /dev/null +++ b/third_party/nix/src/libexpr/common-eval-args.cc @@ -0,0 +1,72 @@ +#include "libexpr/common-eval-args.hh" + +#include "libexpr/eval.hh" +#include "libmain/shared.hh" +#include "libstore/download.hh" +#include "libutil/util.hh" + +namespace nix { + +MixEvalArgs::MixEvalArgs() { + mkFlag() + .longName("arg") + .description("argument to be passed to Nix functions") + .labels({"name", "expr"}) + .handler([&](std::vector<std::string> ss) { + auto_args_[ss[0]] = std::make_pair(kArgTypeExpr, ss[1]); + }); + + mkFlag() + .longName("argstr") + .description("string-valued argument to be passed to Nix functions") + .labels({"name", "string"}) + .handler([&](std::vector<std::string> ss) { + auto_args_[ss[0]] = std::make_pair(kArgTypeString, ss[1]); + }); + + mkFlag() + .shortName('I') + .longName("include") + .description( + "add a path to the list of locations used to look up <...> file " + "names") + .label("path") + .handler([&](const std::string& s) { searchPath.push_back(s); }); +} + +std::unique_ptr<Bindings> MixEvalArgs::getAutoArgs(EvalState& state) { + auto res = Bindings::New(auto_args_.size()); + for (auto& [arg, arg_value] : auto_args_) { + Value* v = state.allocValue(); + switch (arg_value.first) { + case kArgTypeExpr: { + state.mkThunk_( + *v, state.parseExprFromString(arg_value.second, absPath("."))); + break; + } + case kArgTypeString: { + mkString(*v, arg_value.second); + break; + } + } + + res->push_back(Attr(state.symbols.Create(arg), v)); + } + return res; +} + +Path lookupFileArg(EvalState& state, std::string s) { + if (isUri(s)) { + CachedDownloadRequest request(s); + request.unpack = true; + return getDownloader()->downloadCached(state.store, request).path; + } + if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + Path p = s.substr(1, s.size() - 2); + return state.findFile(p); + } else { + return absPath(s); + } +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/common-eval-args.hh b/third_party/nix/src/libexpr/common-eval-args.hh new file mode 100644 index 0000000000..5e0e8af79c --- /dev/null +++ b/third_party/nix/src/libexpr/common-eval-args.hh @@ -0,0 +1,26 @@ +#pragma once + +#include "libutil/args.hh" + +namespace nix { + +class Store; +class EvalState; +class Bindings; + +enum ArgType { kArgTypeString, kArgTypeExpr }; + +struct MixEvalArgs : virtual Args { + MixEvalArgs(); + + std::unique_ptr<Bindings> getAutoArgs(EvalState& state); + + Strings searchPath; + + private: + std::map<std::string, std::pair<ArgType, std::string>> auto_args_; +}; + +Path lookupFileArg(EvalState& state, std::string s); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/eval-inline.hh b/third_party/nix/src/libexpr/eval-inline.hh new file mode 100644 index 0000000000..5162ab3971 --- /dev/null +++ b/third_party/nix/src/libexpr/eval-inline.hh @@ -0,0 +1,90 @@ +#pragma once + +#include "libexpr/eval.hh" + +#define LocalNoInline(f) \ + static f __attribute__((noinline)); \ + f +#define LocalNoInlineNoReturn(f) \ + static f __attribute__((noinline, noreturn)); \ + f + +namespace nix { + +LocalNoInlineNoReturn(void throwEvalError(const char* s, const Pos& pos)) { + throw EvalError(format(s) % pos); +} + +LocalNoInlineNoReturn(void throwTypeError(const char* s, const Value& v)) { + throw TypeError(format(s) % showType(v)); +} + +LocalNoInlineNoReturn(void throwTypeError(const char* s, const Value& v, + const Pos& pos)) { + throw TypeError(format(s) % showType(v) % pos); +} + +void EvalState::forceValue(Value& v, const Pos& pos) { + if (v.type == tThunk) { + Env* env = v.thunk.env; + Expr* expr = v.thunk.expr; + try { + v.type = tBlackhole; + // checkInterrupt(); + expr->eval(*this, *env, v); + } catch (...) { + v.type = tThunk; + v.thunk.env = env; + v.thunk.expr = expr; + throw; + } + } else if (v.type == tApp) { + callFunction(*v.app.left, *v.app.right, v, noPos); + } else if (v.type == tBlackhole) { + throwEvalError("infinite recursion encountered, at %1%", pos); + } +} + +inline void EvalState::forceAttrs(Value& v) { + forceValue(v); + if (v.type != tAttrs) { + throwTypeError("value is %1% while a set was expected", v); + } +} + +inline void EvalState::forceAttrs(Value& v, const Pos& pos) { + forceValue(v); + if (v.type != tAttrs) { + throwTypeError("value is %1% while a set was expected, at %2%", v, pos); + } +} + +inline void EvalState::forceList(Value& v) { + forceValue(v); + if (!v.isList()) { + throwTypeError("value is %1% while a list was expected", v); + } +} + +inline void EvalState::forceList(Value& v, const Pos& pos) { + forceValue(v); + if (!v.isList()) { + throwTypeError("value is %1% while a list was expected, at %2%", v, pos); + } +} + +/* Note: Various places expect the allocated memory to be zeroed. */ +inline void* allocBytes(size_t n) { + void* p; +#if HAVE_BOEHMGC + p = GC_MALLOC(n); +#else + p = calloc(n, 1); +#endif + if (!p) { + throw std::bad_alloc(); + } + return p; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/eval.cc b/third_party/nix/src/libexpr/eval.cc new file mode 100644 index 0000000000..682ea64832 --- /dev/null +++ b/third_party/nix/src/libexpr/eval.cc @@ -0,0 +1,1878 @@ +#include "libexpr/eval.hh" + +#include <algorithm> +#include <chrono> +#include <cstdint> +#include <cstring> +#include <fstream> +#include <iostream> +#include <memory> +#include <new> +#include <optional> +#include <variant> + +#include <absl/base/call_once.h> +#include <absl/container/flat_hash_set.h> +#include <absl/strings/match.h> +#include <glog/logging.h> +#include <sys/resource.h> +#include <sys/time.h> +#include <unistd.h> + +#include "libexpr/eval-inline.hh" +#include "libexpr/function-trace.hh" +#include "libexpr/value.hh" +#include "libstore/derivations.hh" +#include "libstore/download.hh" +#include "libstore/globals.hh" +#include "libstore/store-api.hh" +#include "libutil/hash.hh" +#include "libutil/json.hh" +#include "libutil/util.hh" +#include "libutil/visitor.hh" + +namespace nix { +namespace { + +void ConfigureGc() { /* This function intentionally left blank. */ +} + +} // namespace + +namespace expr { + +absl::once_flag gc_flag; + +void InitGC() { absl::call_once(gc_flag, &ConfigureGc); } + +} // namespace expr + +static char* dupString(const char* s) { + char* t; + t = strdup(s); + if (t == nullptr) { + throw std::bad_alloc(); + } + return t; +} + +std::shared_ptr<Value*> allocRootValue(Value* v) { + return std::make_shared<Value*>(v); +} + +static void printValue(std::ostream& str, std::set<const Value*>& active, + const Value& v) { + checkInterrupt(); + + if (active.find(&v) != active.end()) { + str << "<CYCLE>"; + return; + } + active.insert(&v); + + switch (v.type) { + case tInt: + str << v.integer; + break; + case tBool: + str << (v.boolean ? "true" : "false"); + break; + case tString: + str << "\""; + for (const char* i = v.string.s; *i != 0; i++) { + if (*i == '\"' || *i == '\\') { + str << "\\" << *i; + } else if (*i == '\n') { + str << "\\n"; + } else if (*i == '\r') { + str << "\\r"; + } else if (*i == '\t') { + str << "\\t"; + } else { + str << *i; + } + } + str << "\""; + break; + case tPath: + str << v.path; // !!! escaping? + break; + case tNull: + str << "null"; + break; + case tAttrs: { + str << "{ "; + for (const auto& [key, value] : *v.attrs) { + str << key << " = "; + printValue(str, active, *value.value); + str << "; "; + } + str << "}"; + break; + } + case tList: + str << "[ "; + for (unsigned int n = 0; n < v.listSize(); ++n) { + printValue(str, active, *(*v.list)[n]); + str << " "; + } + str << "]"; + break; + case tThunk: + case tApp: + str << "<CODE>"; + break; + case tLambda: + str << "<LAMBDA>"; + break; + case tPrimOp: + str << "<PRIMOP>"; + break; + case tPrimOpApp: + str << "<PRIMOP-APP>"; + break; + case tFloat: + str << v.fpoint; + break; + default: + throw Error( + absl::StrCat("invalid value of type ", static_cast<int>(v.type))); + } + + active.erase(&v); +} + +std::ostream& operator<<(std::ostream& str, const Value& v) { + std::set<const Value*> active; + printValue(str, active, v); + return str; +} + +const Value* getPrimOp(const Value& v) { + const Value* primOp = &v; + while (primOp->type == tPrimOpApp) { + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + return primOp; +} + +std::string showType(const Value& v) { + switch (v.type) { + case tInt: + return "an integer"; + case tBool: + return "a boolean"; + case tString: + return v.string.context != nullptr ? "a string with context" : "a string"; + case tPath: + return "a path"; + case tNull: + return "null"; + case tAttrs: + return "a set"; + case tList: + return "a list"; + case tThunk: + return "a thunk"; + case tApp: + return "a function application"; + case tLambda: + return "a function"; + case tBlackhole: + return "a black hole"; + case tPrimOp: + return fmt("the built-in function '%s'", std::string(v.primOp->name)); + case tPrimOpApp: + return fmt("the partially applied built-in function '%s'", + std::string(getPrimOp(v)->primOp->name)); + case _reserved1: + LOG(FATAL) << "attempted to show the type string of the deprecated " + "tExternal value"; + break; + case tFloat: + return "a float"; + } + LOG(FATAL) + << "attempted to determine the type string of an unknown type number (" + << static_cast<int>(v.type) << ")"; + abort(); +} + +static Symbol getName(const AttrName& name, EvalState& state, Env& env) { + return std::visit( + util::overloaded{[&](const Symbol& name) -> Symbol { return name; }, + [&](Expr* expr) -> Symbol { + Value nameValue; + expr->eval(state, env, nameValue); + state.forceStringNoCtx(nameValue); + return state.symbols.Create(nameValue.string.s); + }}, + name); +} + +/* Very hacky way to parse $NIX_PATH, which is colon-separated, but + can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ +static Strings parseNixPath(const std::string& s) { + Strings res; + + auto p = s.begin(); + + while (p != s.end()) { + auto start = p; + auto start2 = p; + + while (p != s.end() && *p != ':') { + if (*p == '=') { + start2 = p + 1; + } + ++p; + } + + if (p == s.end()) { + if (p != start) { + res.push_back(std::string(start, p)); + } + break; + } + + if (*p == ':') { + if (isUri(std::string(start2, s.end()))) { + ++p; + while (p != s.end() && *p != ':') { + ++p; + } + } + res.push_back(std::string(start, p)); + if (p == s.end()) { + break; + } + } + + ++p; + } + + return res; +} + +EvalState::EvalState(const Strings& _searchPath, const ref<Store>& store) + : sWith(symbols.Create("<with>")), + sOutPath(symbols.Create("outPath")), + sDrvPath(symbols.Create("drvPath")), + sType(symbols.Create("type")), + sMeta(symbols.Create("meta")), + sName(symbols.Create("name")), + sValue(symbols.Create("value")), + sSystem(symbols.Create("system")), + sOutputs(symbols.Create("outputs")), + sOutputName(symbols.Create("outputName")), + sIgnoreNulls(symbols.Create("__ignoreNulls")), + sFile(symbols.Create("file")), + sLine(symbols.Create("line")), + sColumn(symbols.Create("column")), + sFunctor(symbols.Create("__functor")), + sToString(symbols.Create("__toString")), + sRight(symbols.Create("right")), + sWrong(symbols.Create("wrong")), + sStructuredAttrs(symbols.Create("__structuredAttrs")), + sBuilder(symbols.Create("builder")), + sArgs(symbols.Create("args")), + sOutputHash(symbols.Create("outputHash")), + sOutputHashAlgo(symbols.Create("outputHashAlgo")), + sOutputHashMode(symbols.Create("outputHashMode")), + sDerivationNix(std::nullopt), + repair(NoRepair), + store(store), + baseEnv(allocEnv(128)), + staticBaseEnv(false, nullptr) { + expr::InitGC(); + + countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; + + /* Initialise the Nix expression search path. */ + if (!evalSettings.pureEval) { + Strings paths = parseNixPath(getEnv("NIX_PATH").value_or("")); + for (auto& i : _searchPath) { + addToSearchPath(i); + } + for (auto& i : paths) { + addToSearchPath(i); + } + } + addToSearchPath("nix=" + + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); + + if (evalSettings.restrictEval || evalSettings.pureEval) { + allowedPaths = PathSet(); + + for (auto& i : searchPath) { + auto r = resolveSearchPathElem(i); + if (!r.first) { + continue; + } + + auto path = r.second; + + if (store->isInStore(r.second)) { + PathSet closure; + store->computeFSClosure(store->toStorePath(r.second), closure); + for (auto& path : closure) { + allowedPaths->insert(path); + } + } else { + allowedPaths->insert(r.second); + } + } + } + + createBaseEnv(); +} + +EvalState::~EvalState() = default; + +Path EvalState::checkSourcePath(const Path& path_) { + TraceFileAccess(path_); + if (!allowedPaths) { + return path_; + } + + auto i = resolvedPaths.find(path_); + if (i != resolvedPaths.end()) { + return i->second; + } + + bool found = false; + + /* First canonicalize the path without symlinks, so we make sure an + * attacker can't append ../../... to a path that would be in allowedPaths + * and thus leak symlink targets. + */ + Path abspath = canonPath(path_); + + for (auto& i : *allowedPaths) { + if (isDirOrInDir(abspath, i)) { + found = true; + break; + } + } + + if (!found) { + throw RestrictedPathError( + "access to path '%1%' is forbidden in restricted mode", abspath); + } + + /* Resolve symlinks. */ + DLOG(INFO) << "checking access to '" << abspath << "'"; + Path path = canonPath(abspath, true); + + for (auto& i : *allowedPaths) { + if (isDirOrInDir(path, i)) { + resolvedPaths[path_] = path; + return path; + } + } + + throw RestrictedPathError( + "access to path '%1%' is forbidden in restricted mode", path); +} + +void EvalState::checkURI(const std::string& uri) { + if (!evalSettings.restrictEval) { + return; + } + + /* 'uri' should be equal to a prefix, or in a subdirectory of a + prefix. Thus, the prefix https://github.co does not permit + access to https://github.com. Note: this allows 'http://' and + 'https://' as prefixes for any http/https URI. */ + for (auto& prefix : evalSettings.allowedUris.get()) { + if (uri == prefix || + (uri.size() > prefix.size() && !prefix.empty() && + absl::StartsWith(uri, prefix) && + (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) { + return; + } + } + + /* If the URI is a path, then check it against allowedPaths as + well. */ + if (absl::StartsWith(uri, "/")) { + checkSourcePath(uri); + return; + } + + if (absl::StartsWith(uri, "file://")) { + checkSourcePath(std::string(uri, 7)); + return; + } + + throw RestrictedPathError( + "access to URI '%s' is forbidden in restricted mode", uri); +} + +Path EvalState::toRealPath(const Path& path, const PathSet& context) { + // FIXME: check whether 'path' is in 'context'. + return !context.empty() && store->isInStore(path) ? store->toRealPath(path) + : path; +}; + +Value* EvalState::addConstant(const std::string& name, Value& v) { + Value* v2 = allocValue(); + *v2 = v; + staticBaseEnv.vars[symbols.Create(name)] = baseEnvDispl; + baseEnv.values[baseEnvDispl++] = v2; + std::string name2 = + std::string(name, 0, 2) == "__" ? std::string(name, 2) : name; + baseEnv.values[0]->attrs->push_back(Attr(symbols.Create(name2), v2)); + return v2; +} + +Value* EvalState::addPrimOp(const std::string& name, size_t arity, + PrimOpFun primOp) { + if (arity == 0) { + Value v; + primOp(*this, noPos, nullptr, v); + return addConstant(name, v); + } + std::string name2 = + std::string(name, 0, 2) == "__" ? std::string(name, 2) : name; + Symbol sym = symbols.Create(name2); + Value* v = allocValue(); + v->type = tPrimOp; + v->primOp = std::make_shared<PrimOp>(primOp, arity, sym); + staticBaseEnv.vars[symbols.Create(name)] = baseEnvDispl; + baseEnv.values[baseEnvDispl++] = v; + baseEnv.values[0]->attrs->push_back(Attr(sym, v)); + return v; +} + +Value& EvalState::getBuiltin(const std::string& name) { + return *baseEnv.values[0]->attrs->find(symbols.Create(name))->second.value; +} + +/* Every "format" object (even temporary) takes up a few hundred bytes + of stack space, which is a real killer in the recursive + evaluator. So here are some helper functions for throwing + exceptions. */ + +LocalNoInlineNoReturn(void throwEvalError(const char* s, + const std::string& s2)) { + throw EvalError(format(s) % s2); +} + +LocalNoInlineNoReturn(void throwEvalError(const char* s, const std::string& s2, + const Pos& pos)) { + throw EvalError(format(s) % s2 % pos); +} + +LocalNoInlineNoReturn(void throwEvalError(const char* s, const std::string& s2, + const std::string& s3)) { + throw EvalError(format(s) % s2 % s3); +} + +LocalNoInlineNoReturn(void throwEvalError(const char* s, const std::string& s2, + const std::string& s3, + const Pos& pos)) { + throw EvalError(format(s) % s2 % s3 % pos); +} + +LocalNoInlineNoReturn(void throwEvalError(const char* s, const Symbol& sym, + const Pos& p1, const Pos& p2)) { + throw EvalError(format(s) % sym % p1 % p2); +} + +LocalNoInlineNoReturn(void throwTypeError(const char* s, const Pos& pos)) { + throw TypeError(format(s) % pos); +} + +LocalNoInlineNoReturn(void throwTypeError(const char* s, + const std::string& s1)) { + throw TypeError(format(s) % s1); +} + +LocalNoInlineNoReturn(void throwTypeError(const char* s, const ExprLambda& fun, + const Symbol& s2, const Pos& pos)) { + throw TypeError(format(s) % fun.showNamePos() % s2 % pos); +} + +LocalNoInlineNoReturn(void throwAssertionError(const char* s, + const std::string& s1, + const Pos& pos)) { + throw AssertionError(format(s) % s1 % pos); +} + +LocalNoInlineNoReturn(void throwUndefinedVarError(const char* s, + const std::string& s1, + const Pos& pos)) { + throw UndefinedVarError(format(s) % s1 % pos); +} + +LocalNoInline(void addErrorPrefix(Error& e, const char* s, + const std::string& s2)) { + e.addPrefix(format(s) % s2); +} + +LocalNoInline(void addErrorPrefix(Error& e, const char* s, + const ExprLambda& fun, const Pos& pos)) { + e.addPrefix(format(s) % fun.showNamePos() % pos); +} + +LocalNoInline(void addErrorPrefix(Error& e, const char* s, + const std::string& s2, const Pos& pos)) { + e.addPrefix(format(s) % s2 % pos); +} + +void mkString(Value& v, const char* s) { mkStringNoCopy(v, dupString(s)); } + +Value& mkString(Value& v, const std::string& s, const PathSet& context) { + mkString(v, s.c_str()); + if (!context.empty()) { + size_t n = 0; + v.string.context = static_cast<const char**>( + allocBytes((context.size() + 1) * sizeof(char*))); + for (auto& i : context) { + v.string.context[n++] = dupString(i.c_str()); + } + v.string.context[n] = nullptr; + } + return v; +} + +void mkPath(Value& v, const char* s) { mkPathNoCopy(v, dupString(s)); } + +inline Value* EvalState::lookupVar(Env* env, const ExprVar& var, bool noEval) { + for (size_t l = var.level; l != 0u; --l, env = env->up) { + ; + } + + if (!var.fromWith) { + return env->values[var.displ]; + } + + while (true) { + if (env->type == Env::HasWithExpr) { + if (noEval) { + return nullptr; + } + if (!env->withAttrsExpr) { + CHECK(false) << "HasWithExpr evaluated twice"; + } + Value* v = allocValue(); + evalAttrs(*env->up, env->withAttrsExpr, *v); + env->values[0] = v; + env->withAttrsExpr = nullptr; + env->type = Env::HasWithAttrs; + } + Bindings::iterator j = env->values[0]->attrs->find(var.name); + if (j != env->values[0]->attrs->end()) { + if (countCalls && (j->second.pos != nullptr)) { + attrSelects[*j->second.pos]++; + } + return j->second.value; + } + if (env->prevWith == 0u) { + throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, + var.pos); + } + for (size_t l = env->prevWith; l != 0u; --l, env = env->up) { + } + } +} + +Value* EvalState::allocValue() { + nrValues++; + return new Value; +} + +Env& EvalState::allocEnv(size_t size) { + if (size > std::numeric_limits<decltype(Env::size)>::max()) { + throw Error("environment size %d is too big", size); + } + + nrEnvs++; + nrValuesInEnvs += size; + Env* env = new Env(size); + env->type = Env::Plain; + + return *env; +} + +void EvalState::mkList(Value& v, std::shared_ptr<NixList> list) { + nrListElems += list->size(); + clearValue(v); + v.type = tList; + v.list = list; +} + +void EvalState::mkList(Value& v, size_t size) { + EvalState::mkList(v, std::make_shared<NixList>(size)); +} + +unsigned long nrThunks = 0; + +static inline void mkThunk(Value& v, Env& env, Expr* expr) { + v.type = tThunk; + v.thunk.env = &env; + v.thunk.expr = expr; + nrThunks++; +} + +void EvalState::mkThunk_(Value& v, Expr* expr) { mkThunk(v, baseEnv, expr); } + +void EvalState::mkPos(Value& v, Pos* pos) { + if ((pos != nullptr) && pos->file.has_value() && pos->file.value().set()) { + mkAttrs(v, 3); + mkString(*allocAttr(v, sFile), pos->file.value()); + mkInt(*allocAttr(v, sLine), pos->line); + mkInt(*allocAttr(v, sColumn), pos->column); + } else { + mkNull(v); + } +} + +/* Create a thunk for the delayed computation of the given expression + in the given environment. But if the expression is a variable, + then look it up right away. This significantly reduces the number + of thunks allocated. */ +Value* Expr::maybeThunk(EvalState& state, Env& env) { + Value* v = state.allocValue(); + mkThunk(*v, env, this); + return v; +} + +unsigned long nrAvoided = 0; + +Value* ExprVar::maybeThunk(EvalState& state, Env& env) { + Value* v = state.lookupVar(&env, *this, true); + /* The value might not be initialised in the environment yet. + In that case, ignore it. */ + if (v != nullptr) { + nrAvoided++; + return v; + } + return Expr::maybeThunk(state, env); +} + +Value* ExprString::maybeThunk(EvalState& state, Env& env) { + nrAvoided++; + return &v; +} + +Value* ExprInt::maybeThunk(EvalState& state, Env& env) { + nrAvoided++; + return &v; +} + +Value* ExprFloat::maybeThunk(EvalState& state, Env& env) { + nrAvoided++; + return &v; +} + +Value* ExprPath::maybeThunk(EvalState& state, Env& env) { + nrAvoided++; + return &v; +} + +void EvalState::evalFile(const Path& path_, Value& v) { + auto path = checkSourcePath(path_); + + FileEvalCache::iterator i; + if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) { + v = i->second; + return; + } + + Path path2 = resolveExprPath(path); + if ((i = fileEvalCache.find(path2)) != fileEvalCache.end()) { + v = i->second; + return; + } + + VLOG(2) << "evaluating file '" << path2 << "'"; + Expr* e = nullptr; + + auto j = fileParseCache.find(path2); + if (j != fileParseCache.end()) { + e = j->second; + } + + if (e == nullptr) { + e = parseExprFromFile(checkSourcePath(path2)); + } + + fileParseCache[path2] = e; + + try { + eval(e, v); + } catch (Error& e) { + addErrorPrefix(e, "while evaluating the file '%1%':\n", path2); + throw; + } + + fileEvalCache[path2] = v; + if (path != path2) { + fileEvalCache[path] = v; + } +} + +void EvalState::resetFileCache() { + fileEvalCache.clear(); + fileParseCache.clear(); +} + +void EvalState::eval(Expr* e, Value& v) { e->eval(*this, baseEnv, v); } + +inline bool EvalState::evalBool(Env& env, Expr* e) { + Value v; + e->eval(*this, env, v); + if (v.type != tBool) { + throwTypeError("value is %1% while a Boolean was expected", v); + } + return v.boolean; +} + +inline bool EvalState::evalBool(Env& env, Expr* e, const Pos& pos) { + Value v; + e->eval(*this, env, v); + if (v.type != tBool) { + throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); + } + return v.boolean; +} + +inline void EvalState::evalAttrs(Env& env, Expr* e, Value& v) { + e->eval(*this, env, v); + if (v.type != tAttrs) { + throwTypeError("value is %1% while a set was expected", v); + } +} + +void Expr::eval(EvalState& state, Env& env, Value& v) { abort(); } + +void ExprInt::eval(EvalState& state, Env& env, Value& v) { v = this->v; } + +void ExprFloat::eval(EvalState& state, Env& env, Value& v) { v = this->v; } + +void ExprString::eval(EvalState& state, Env& env, Value& v) { v = this->v; } + +void ExprPath::eval(EvalState& state, Env& env, Value& v) { v = this->v; } + +void ExprAttrs::eval(EvalState& state, Env& env, Value& value) { + state.mkAttrs(value, attrs.size() + dynamicAttrs.size()); + Env* dynamicEnv = &env; + + if (recursive) { + /* Create a new environment that contains the attributes in + this `rec'. */ + Env& env2(state.allocEnv(attrs.size())); + env2.up = &env; + dynamicEnv = &env2; + + /* The recursive attributes are evaluated in the new + environment, while the inherited attributes are evaluated + in the original environment. */ + size_t displ = 0; + for (auto& attr : attrs) { + Value* vAttr; + vAttr = + attr.second.e->maybeThunk(state, attr.second.inherited ? env : env2); + env2.values[displ++] = vAttr; + value.attrs->push_back(Attr(attr.first, vAttr, &attr.second.pos)); + } + } else { + // TODO(tazjin): insert range + for (auto& i : attrs) { + value.attrs->push_back( + Attr(i.first, i.second.e->maybeThunk(state, env), &i.second.pos)); + } + } + + /* Dynamic attrs apply *after* rec. */ + for (auto& i : dynamicAttrs) { + Value nameVal; + i.nameExpr->eval(state, *dynamicEnv, nameVal); + state.forceValue(nameVal, i.pos); + if (nameVal.type == tNull) { + continue; + } + state.forceStringNoCtx(nameVal); + Symbol nameSym = state.symbols.Create(nameVal.string.s); + Bindings::iterator j = value.attrs->find(nameSym); + if (j != value.attrs->end()) { + throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", + nameSym, i.pos, *j->second.pos); + } + + value.attrs->push_back( + Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos)); + } +} + +void ExprLet::eval(EvalState& state, Env& env, Value& v) { + /* Create a new environment that contains the attributes in this + `let'. */ + Env& env2(state.allocEnv(attrs->attrs.size())); + env2.up = &env; + + /* The recursive attributes are evaluated in the new environment, + while the inherited attributes are evaluated in the original + environment. */ + size_t displ = 0; + for (auto& i : attrs->attrs) { + env2.values[displ++] = + i.second.e->maybeThunk(state, i.second.inherited ? env : env2); + } + + body->eval(state, env2, v); +} + +void ExprList::eval(EvalState& state, Env& env, Value& v) { + state.mkList(v, elems.size()); + for (size_t n = 0; n < elems.size(); ++n) { + (*v.list)[n] = elems[n]->maybeThunk(state, env); + } +} + +void ExprVar::eval(EvalState& state, Env& env, Value& v) { + Value* v2 = state.lookupVar(&env, *this, false); + state.forceValue(*v2, pos); + v = *v2; +} + +static std::string showAttrPath(EvalState& state, Env& env, + const AttrPath& attrPath) { + std::ostringstream out; + bool first = true; + for (auto& i : attrPath) { + if (!first) { + out << '.'; + } else { + first = false; + } + out << getName(i, state, env); + } + return out.str(); +} + +uint64_t nrLookups = 0; + +void ExprSelect::eval(EvalState& state, Env& env, Value& v) { + Value vTmp; + Pos* pos2 = nullptr; + Value* vAttrs = &vTmp; + + e->eval(state, env, vTmp); + + try { + for (auto& i : attrPath) { + nrLookups++; + Bindings::iterator j; + Symbol name = getName(i, state, env); + if (def != nullptr) { + state.forceValue(*vAttrs, pos); + if (vAttrs->type != tAttrs || + (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { + def->eval(state, env, v); + return; + } + } else { + state.forceAttrs(*vAttrs, pos); + if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { + throwEvalError("attribute '%1%' missing, at %2%", name, pos); + } + } + vAttrs = j->second.value; + pos2 = j->second.pos; + if (state.countCalls && (pos2 != nullptr)) { + state.attrSelects[*pos2]++; + } + } + + state.forceValue(*vAttrs, (pos2 != nullptr ? *pos2 : this->pos)); + + } catch (Error& e) { + // This code relies on 'sDerivationNix' being correcty mutated at + // some prior point (it would previously otherwise have been a + // nullptr). + // + // We haven't seen this fail, so for now the contained value is + // just accessed at the risk of potentially crashing. + if ((pos2 != nullptr) && pos2->file != state.sDerivationNix.value()) { + addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", + showAttrPath(state, env, attrPath), *pos2); + } + throw; + } + + v = *vAttrs; +} + +void ExprOpHasAttr::eval(EvalState& state, Env& env, Value& v) { + Value vTmp; + Value* vAttrs = &vTmp; + + e->eval(state, env, vTmp); + + for (auto& i : attrPath) { + state.forceValue(*vAttrs); + Bindings::iterator j; + Symbol name = getName(i, state, env); + if (vAttrs->type != tAttrs || + (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { + mkBool(v, false); + return; + } + vAttrs = j->second.value; + } + + mkBool(v, true); +} + +void ExprLambda::eval(EvalState& state, Env& env, Value& v) { + v.type = tLambda; + v.lambda.env = &env; + v.lambda.fun = this; +} + +void ExprApp::eval(EvalState& state, Env& env, Value& v) { + /* FIXME: vFun prevents GCC from doing tail call optimisation. */ + Value vFun; + e1->eval(state, env, vFun); + state.callFunction(vFun, *(e2->maybeThunk(state, env)), v, pos); +} + +void EvalState::callPrimOp(Value& fun, Value& arg, Value& v, const Pos& pos) { + /* Figure out the number of arguments still needed. */ + size_t argsDone = 0; + Value* primOp = &fun; + while (primOp->type == tPrimOpApp) { + argsDone++; + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + auto arity = primOp->primOp->arity; + auto argsLeft = arity - argsDone; + + if (argsLeft == 1) { + /* We have all the arguments, so call the primop. */ + + /* Put all the arguments in an array. */ + Value* vArgs[arity]; + auto n = arity - 1; + vArgs[n--] = &arg; + for (Value* arg = &fun; arg->type == tPrimOpApp; + arg = arg->primOpApp.left) { + vArgs[n--] = arg->primOpApp.right; + } + + /* And call the primop. */ + nrPrimOpCalls++; + if (countCalls) { + primOpCalls[primOp->primOp->name]++; + } + primOp->primOp->fun(*this, pos, vArgs, v); + } else { + Value* fun2 = allocValue(); + *fun2 = fun; + v.type = tPrimOpApp; + v.primOpApp.left = fun2; + v.primOpApp.right = &arg; + } +} + +void EvalState::callFunction(Value& fun, Value& arg, Value& v, const Pos& pos) { + auto trace = evalSettings.traceFunctionCalls + ? std::make_unique<FunctionCallTrace>(pos) + : nullptr; + + forceValue(fun, pos); + + if (fun.type == tPrimOp || fun.type == tPrimOpApp) { + callPrimOp(fun, arg, v, pos); + return; + } + + // If the value to be called is an attribute set, check whether it + // contains an appropriate function in the '__functor' element and + // use that. + if (fun.type == tAttrs) { + auto found = fun.attrs->find(sFunctor); + if (found != fun.attrs->end()) { + // fun may be allocated on the stack of the calling function, + // but for functors we may keep a reference, so heap-allocate a + // copy and use that instead + auto& fun2 = *allocValue(); + fun2 = fun; + /* !!! Should we use the attr pos here? */ + Value v2; + // functors are called with the element itself as the first + // parameter, which is partially applied here + callFunction(*found->second.value, fun2, v2, pos); + return callFunction(v2, arg, v, pos); + } + } + + if (fun.type != tLambda) { + throwTypeError( + "attempt to call something which is not a function but %1%, at %2%", + fun, pos); + } + + ExprLambda& lambda(*fun.lambda.fun); + + auto size = (lambda.arg.empty() ? 0 : 1) + + (lambda.matchAttrs ? lambda.formals->formals.size() : 0); + Env& env2(allocEnv(size)); + env2.up = fun.lambda.env; + + size_t displ = 0; + + if (!lambda.matchAttrs) { + env2.values[displ++] = &arg; + + } else { + forceAttrs(arg, pos); + + if (!lambda.arg.empty()) { + env2.values[displ++] = &arg; + } + + /* For each formal argument, get the actual argument. If + there is no matching actual argument but the formal + argument has a default, use the default. */ + size_t attrsUsed = 0; + for (auto& i : lambda.formals->formals) { + Bindings::iterator j = arg.attrs->find(i.name); + if (j == arg.attrs->end()) { + if (i.def == nullptr) { + throwTypeError("%1% called without required argument '%2%', at %3%", + lambda, i.name, pos); + } + env2.values[displ++] = i.def->maybeThunk(*this, env2); + } else { + attrsUsed++; + env2.values[displ++] = j->second.value; + } + } + + /* Check that each actual argument is listed as a formal + argument (unless the attribute match specifies a `...'). */ + if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) { + /* Nope, so show the first unexpected argument to the + user. */ + for (auto& i : *arg.attrs) { + if (lambda.formals->argNames.find(i.second.name) == + lambda.formals->argNames.end()) { + throwTypeError("%1% called with unexpected argument '%2%', at %3%", + lambda, i.second.name, pos); + } + } + abort(); // shouldn't happen + } + } + + nrFunctionCalls++; + if (countCalls) { + incrFunctionCall(&lambda); + } + + /* Evaluate the body. This is conditional on showTrace, because + catching exceptions makes this function not tail-recursive. */ + if (settings.showTrace) { + try { + lambda.body->eval(*this, env2, v); + } catch (Error& e) { + addErrorPrefix(e, "while evaluating %1%, called from %2%:\n", lambda, + pos); + throw; + } + } else { + fun.lambda.fun->body->eval(*this, env2, v); + } +} + +// Lifted out of callFunction() because it creates a temporary that +// prevents tail-call optimisation. +void EvalState::incrFunctionCall(ExprLambda* fun) { functionCalls[fun]++; } + +void EvalState::autoCallFunction(Bindings* args, Value& fun, Value& res) { + forceValue(fun); + + if (fun.type == tAttrs) { + auto found = fun.attrs->find(sFunctor); + if (found != fun.attrs->end()) { + Value* v = allocValue(); + callFunction(*found->second.value, fun, *v, noPos); + forceValue(*v); + return autoCallFunction(args, *v, res); + } + } + + if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) { + res = fun; + return; + } + + Value* actualArgs = allocValue(); + mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size()); + + if (fun.lambda.fun->formals->ellipsis) { + // If the formals have an ellipsis (eg the function accepts extra args) pass + // all available automatic arguments (which includes arguments specified on + // the command line via --arg/--argstr) + for (auto& [_, v] : *args) { + actualArgs->attrs->push_back(v); + } + } else { + // Otherwise, only pass the arguments that the function accepts + for (auto& i : fun.lambda.fun->formals->formals) { + Bindings::iterator j = args->find(i.name); + if (j != args->end()) { + actualArgs->attrs->push_back(j->second); + } else if (i.def == nullptr) { + throwTypeError( + "cannot auto-call a function that has an argument without a " + "default " + "value ('%1%')", + i.name); + } + } + } + + callFunction(fun, *actualArgs, res, noPos); +} + +void ExprWith::eval(EvalState& state, Env& env, Value& v) { + Env& env2(state.allocEnv(1)); + env2.up = &env; + env2.prevWith = prevWith; + env2.type = Env::HasWithExpr; + /* placeholder for result of attrs */ + env2.values[0] = nullptr; + env2.withAttrsExpr = this->attrs; + + body->eval(state, env2, v); +} + +void ExprIf::eval(EvalState& state, Env& env, Value& v) { + (state.evalBool(env, cond) ? then : else_)->eval(state, env, v); +} + +void ExprAssert::eval(EvalState& state, Env& env, Value& v) { + if (!state.evalBool(env, cond, pos)) { + std::ostringstream out; + cond->show(out); + throwAssertionError("assertion %1% failed at %2%", out.str(), pos); + } + body->eval(state, env, v); +} + +void ExprOpNot::eval(EvalState& state, Env& env, Value& v) { + mkBool(v, !state.evalBool(env, e)); +} + +void ExprOpEq::eval(EvalState& state, Env& env, Value& v) { + Value v1; + e1->eval(state, env, v1); + Value v2; + e2->eval(state, env, v2); + mkBool(v, state.eqValues(v1, v2)); +} + +void ExprOpNEq::eval(EvalState& state, Env& env, Value& v) { + Value v1; + e1->eval(state, env, v1); + Value v2; + e2->eval(state, env, v2); + mkBool(v, !state.eqValues(v1, v2)); +} + +void ExprOpAnd::eval(EvalState& state, Env& env, Value& v) { + mkBool(v, state.evalBool(env, e1, pos) && state.evalBool(env, e2, pos)); +} + +void ExprOpOr::eval(EvalState& state, Env& env, Value& v) { + mkBool(v, state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); +} + +void ExprOpImpl::eval(EvalState& state, Env& env, Value& v) { + mkBool(v, !state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); +} + +void ExprOpUpdate::eval(EvalState& state, Env& env, Value& dest) { + Value v1; + Value v2; + state.evalAttrs(env, e1, v1); + state.evalAttrs(env, e2, v2); + + state.nrOpUpdates++; + + clearValue(dest); + dest.type = tAttrs; + dest.attrs = Bindings::Merge(*v1.attrs, *v2.attrs); +} + +void ExprOpConcatLists::eval(EvalState& state, Env& env, Value& v) { + Value v1; + e1->eval(state, env, v1); + Value v2; + e2->eval(state, env, v2); + state.concatLists(v, {&v1, &v2}, pos); +} + +void EvalState::concatLists(Value& v, const NixList& lists, const Pos& pos) { + nrListConcats++; + + auto outlist = std::make_shared<NixList>(); + + for (Value* list : lists) { + forceList(*list, pos); + outlist->insert(outlist->end(), list->list->begin(), list->list->end()); + } + + mkList(v, outlist); +} + +void ExprConcatStrings::eval(EvalState& state, Env& env, Value& v) { + PathSet context; + std::ostringstream s; + NixInt n = 0; + NixFloat nf = 0; + + bool first = !forceString; + ValueType firstType = tString; + + for (auto& i : *es) { + Value vTmp; + i->eval(state, env, vTmp); + + /* If the first element is a path, then the result will also + be a path, we don't copy anything (yet - that's done later, + since paths are copied when they are used in a derivation), + and none of the strings are allowed to have contexts. */ + if (first) { + firstType = vTmp.type; + first = false; + } + + if (firstType == tInt) { + if (vTmp.type == tInt) { + n += vTmp.integer; + } else if (vTmp.type == tFloat) { + // Upgrade the type from int to float; + firstType = tFloat; + nf = n; + nf += vTmp.fpoint; + } else { + throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), + pos); + } + } else if (firstType == tFloat) { + if (vTmp.type == tInt) { + nf += vTmp.integer; + } else if (vTmp.type == tFloat) { + nf += vTmp.fpoint; + } else { + throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), + pos); + } + } else { + s << state.coerceToString(pos, vTmp, context, false, + firstType == tString); + } + } + + if (firstType == tInt) { + mkInt(v, n); + } else if (firstType == tFloat) { + mkFloat(v, nf); + } else if (firstType == tPath) { + if (!context.empty()) { + throwEvalError( + "a string that refers to a store path cannot be appended to a path, " + "at %1%", + pos); + } + auto path = canonPath(s.str()); + mkPath(v, path.c_str()); + } else { + mkString(v, s.str(), context); + } +} + +void ExprPos::eval(EvalState& state, Env& env, Value& v) { + state.mkPos(v, &pos); +} + +template <typename T> +using traceable_flat_hash_set = absl::flat_hash_set<T>; + +void EvalState::forceValueDeep(Value& v) { + traceable_flat_hash_set<const Value*> seen; + + std::function<void(Value & v)> recurse; + + recurse = [&](Value& v) { + if (seen.find(&v) != seen.end()) { + return; + } + seen.insert(&v); + + forceValue(v); + + if (v.type == tAttrs) { + for (auto& i : *v.attrs) { + try { + recurse(*i.second.value); + } catch (Error& e) { + addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", + i.second.name, *i.second.pos); + throw; + } + } + } else if (v.isList()) { + for (size_t n = 0; n < v.listSize(); ++n) { + recurse(*(*v.list)[n]); + } + } + }; + + recurse(v); +} + +NixInt EvalState::forceInt(Value& v, const Pos& pos) { + forceValue(v, pos); + if (v.type != tInt) { + throwTypeError("value is %1% while an integer was expected, at %2%", v, + pos); + } + return v.integer; +} + +NixFloat EvalState::forceFloat(Value& v, const Pos& pos) { + forceValue(v, pos); + if (v.type == tInt) { + return static_cast<NixFloat>(v.integer); + } + if (v.type != tFloat) { + throwTypeError("value is %1% while a float was expected, at %2%", v, pos); + } + return v.fpoint; +} + +bool EvalState::forceBool(Value& v, const Pos& pos) { + forceValue(v); + if (v.type != tBool) { + throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); + } + return v.boolean; +} + +bool EvalState::isFunctor(Value& fun) { + return fun.type == tAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); +} + +void EvalState::forceFunction(Value& v, const Pos& pos) { + forceValue(v); + if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && + !isFunctor(v)) { + throwTypeError("value is %1% while a function was expected, at %2%", v, + pos); + } +} + +std::string EvalState::forceString(Value& v, const Pos& pos) { + forceValue(v, pos); + if (v.type != tString) { + if (pos) { + throwTypeError("value is %1% while a string was expected, at %2%", v, + pos); + } else { + throwTypeError("value is %1% while a string was expected", v); + } + } + return std::string(v.string.s); +} + +void copyContext(const Value& v, PathSet& context) { + if (v.string.context != nullptr) { + for (const char** p = v.string.context; *p != nullptr; ++p) { + context.insert(*p); + } + } +} + +std::string EvalState::forceString(Value& v, PathSet& context, const Pos& pos) { + std::string s = forceString(v, pos); + copyContext(v, context); + return s; +} + +std::string EvalState::forceStringNoCtx(Value& v, const Pos& pos) { + std::string s = forceString(v, pos); + if (v.string.context != nullptr) { + if (pos) { + throwEvalError( + "the string '%1%' is not allowed to refer to a store path (such as " + "'%2%'), at %3%", + v.string.s, v.string.context[0], pos); + } else { + throwEvalError( + "the string '%1%' is not allowed to refer to a store path (such as " + "'%2%')", + v.string.s, v.string.context[0]); + } + } + return s; +} + +bool EvalState::isDerivation(Value& v) { + if (v.type != tAttrs) { + return false; + } + Bindings::iterator i = v.attrs->find(sType); + if (i == v.attrs->end()) { + return false; + } + forceValue(*i->second.value); + if (i->second.value->type != tString) { + return false; + } + return strcmp(i->second.value->string.s, "derivation") == 0; +} + +std::optional<std::string> EvalState::tryAttrsToString(const Pos& pos, Value& v, + PathSet& context, + bool coerceMore, + bool copyToStore) { + auto i = v.attrs->find(sToString); + if (i != v.attrs->end()) { + Value v1; + callFunction(*i->second.value, v, v1, pos); + return coerceToString(pos, v1, context, coerceMore, copyToStore); + } + + return {}; +} + +std::string EvalState::coerceToString(const Pos& pos, Value& v, + PathSet& context, bool coerceMore, + bool copyToStore) { + forceValue(v); + + std::string s; + + if (v.type == tString) { + copyContext(v, context); + return v.string.s; + } + + if (v.type == tPath) { + Path path(canonPath(v.path)); + return copyToStore ? copyPathToStore(context, path) : path; + } + + if (v.type == tAttrs) { + auto maybeString = + tryAttrsToString(pos, v, context, coerceMore, copyToStore); + if (maybeString) { + return *maybeString; + } + auto i = v.attrs->find(sOutPath); + if (i == v.attrs->end()) { + throwTypeError("cannot coerce a set to a string, at %1%", pos); + } + return coerceToString(pos, *i->second.value, context, coerceMore, + copyToStore); + } + + if (coerceMore) { + /* Note that `false' is represented as an empty string for + shell scripting convenience, just like `null'. */ + if (v.type == tBool && v.boolean) { + return "1"; + } + if (v.type == tBool && !v.boolean) { + return ""; + } + if (v.type == tInt) { + return std::to_string(v.integer); + } + if (v.type == tFloat) { + return std::to_string(v.fpoint); + } + if (v.type == tNull) { + return ""; + } + + if (v.isList()) { + std::string result; + for (size_t n = 0; n < v.listSize(); ++n) { + result += coerceToString(pos, *(*v.list)[n], context, coerceMore, + copyToStore); + if (n < v.listSize() - 1 + /* !!! not quite correct */ + && (!(*v.list)[n]->isList() || (*v.list)[n]->listSize() != 0)) { + result += " "; + } + } + return result; + } + } + + throwTypeError("cannot coerce %1% to a string, at %2%", v, pos); +} + +std::string EvalState::copyPathToStore(PathSet& context, const Path& path) { + if (nix::isDerivation(path)) { + throwEvalError("file names are not allowed to end in '%1%'", drvExtension); + } + + Path dstPath; + if (!srcToStore[path].empty()) { + dstPath = srcToStore[path]; + } else { + dstPath = + settings.readOnlyMode + ? store + ->computeStorePathForPath(baseNameOf(path), + checkSourcePath(path)) + .first + : store->addToStore(baseNameOf(path), checkSourcePath(path), true, + htSHA256, defaultPathFilter, repair); + srcToStore[path] = dstPath; + VLOG(2) << "copied source '" << path << "' -> '" << dstPath << "'"; + } + + context.insert(dstPath); + return dstPath; +} + +Path EvalState::coerceToPath(const Pos& pos, Value& v, PathSet& context) { + std::string path = coerceToString(pos, v, context, false, false); + if (path.empty() || path[0] != '/') { + throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", + path, pos); + } + return path; +} + +bool EvalState::eqValues(Value& v1, Value& v2) { + forceValue(v1); + forceValue(v2); + + /* !!! Hack to support some old broken code that relies on pointer + equality tests between sets. (Specifically, builderDefs calls + uniqList on a list of sets.) Will remove this eventually. */ + if (&v1 == &v2) { + return true; + } + + // Special case type-compatibility between float and int + if (v1.type == tInt && v2.type == tFloat) { + return v1.integer == v2.fpoint; + } + if (v1.type == tFloat && v2.type == tInt) { + return v1.fpoint == v2.integer; + } + + // All other types are not compatible with each other. + if (v1.type != v2.type) { + return false; + } + + switch (v1.type) { + case tInt: + return v1.integer == v2.integer; + + case tBool: + return v1.boolean == v2.boolean; + + case tString: + return strcmp(v1.string.s, v2.string.s) == 0; + + case tPath: + return strcmp(v1.path, v2.path) == 0; + + case tNull: + return true; + + case tList: + if (v1.listSize() != v2.listSize()) { + return false; + } + for (size_t n = 0; n < v1.listSize(); ++n) { + if (!eqValues(*(*v1.list)[n], *(*v2.list)[n])) { + return false; + } + } + return true; + + case tAttrs: { + // As an optimisation if both values are pointing towards the + // same attribute set, we can skip all this extra work. + if (v1.attrs == v2.attrs) { + return true; + } + + /* If both sets denote a derivation (type = "derivation"), + then compare their outPaths. */ + if (isDerivation(v1) && isDerivation(v2)) { + Bindings::iterator i = v1.attrs->find(sOutPath); + Bindings::iterator j = v2.attrs->find(sOutPath); + if (i != v1.attrs->end() && j != v2.attrs->end()) { + return eqValues(*i->second.value, *j->second.value); + } + } + + return v1.attrs->Equal(v2.attrs.get(), *this); + } + + /* Functions are incomparable. */ + case tLambda: + case tPrimOp: + case tPrimOpApp: + return false; + + case tFloat: + return v1.fpoint == v2.fpoint; + + default: + throwEvalError("cannot compare %1% with %2%", showType(v1), showType(v2)); + } +} + +void EvalState::printStats() { + bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0"; + + struct rusage buf; + getrusage(RUSAGE_SELF, &buf); + float cpuTime = buf.ru_utime.tv_sec + + (static_cast<float>(buf.ru_utime.tv_usec) / 1000000); + + uint64_t bEnvs = nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value*); + uint64_t bLists = nrListElems * sizeof(Value*); + uint64_t bValues = nrValues * sizeof(Value); + uint64_t bAttrsets = + nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr); + + if (showStats) { + auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); + std::fstream fs; + if (outPath != "-") { + fs.open(outPath, std::fstream::out); + } + JSONObject topObj(outPath == "-" ? std::cerr : fs, true); + topObj.attr("cpuTime", cpuTime); + { + auto envs = topObj.object("envs"); + envs.attr("number", nrEnvs); + envs.attr("elements", nrValuesInEnvs); + envs.attr("bytes", bEnvs); + } + { + auto lists = topObj.object("list"); + lists.attr("elements", nrListElems); + lists.attr("bytes", bLists); + lists.attr("concats", nrListConcats); + } + { + auto values = topObj.object("values"); + values.attr("number", nrValues); + values.attr("bytes", bValues); + } + { + auto syms = topObj.object("symbols"); + syms.attr("number", symbols.Size()); + syms.attr("bytes", symbols.TotalSize()); + } + { + auto sets = topObj.object("sets"); + sets.attr("number", nrAttrsets); + sets.attr("bytes", bAttrsets); + sets.attr("elements", nrAttrsInAttrsets); + } + { + auto sizes = topObj.object("sizes"); + sizes.attr("Env", sizeof(Env)); + sizes.attr("Value", sizeof(Value)); + sizes.attr("Bindings", sizeof(Bindings)); + sizes.attr("Attr", sizeof(Attr)); + } + topObj.attr("nrOpUpdates", nrOpUpdates); + topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied); + topObj.attr("nrThunks", nrThunks); + topObj.attr("nrAvoided", nrAvoided); + topObj.attr("nrLookups", nrLookups); + topObj.attr("nrPrimOpCalls", nrPrimOpCalls); + topObj.attr("nrFunctionCalls", nrFunctionCalls); + + if (countCalls) { + { + auto obj = topObj.object("primops"); + for (auto& i : primOpCalls) { + obj.attr(i.first, i.second); + } + } + { + auto list = topObj.list("functions"); + for (auto& i : functionCalls) { + auto obj = list.object(); + if (i.first->name.has_value()) { + obj.attr("name", (const std::string&)i.first->name.value()); + } else { + obj.attr("name", nullptr); + } + if (i.first->pos) { + obj.attr("file", (const std::string&)i.first->pos.file); + obj.attr("line", i.first->pos.line); + obj.attr("column", i.first->pos.column); + } + obj.attr("count", i.second); + } + } + { + auto list = topObj.list("attributes"); + for (auto& i : attrSelects) { + auto obj = list.object(); + if (i.first) { + obj.attr("file", (const std::string&)i.first.file); + obj.attr("line", i.first.line); + obj.attr("column", i.first.column); + } + obj.attr("count", i.second); + } + } + } + + // TODO(tazjin): what is this? commented out because .dump() is gone. + // if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") { + // auto list = topObj.list("symbols"); + // symbols.dump([&](const std::string& s) { list.elem(s); }); + // } + } +} + +void EvalState::TraceFileAccess(const Path& realPath) { + if (file_access_trace_fn) { + if (last_traced_file != realPath) { + file_access_trace_fn(realPath); + // Basic deduplication. + last_traced_file = std::string(realPath); + } + } +} + +void EvalState::EnableFileAccessTracing(std::function<void(const Path&)> fn) { + file_access_trace_fn = fn; +} + +size_t valueSize(const Value& v) { + traceable_flat_hash_set<const Bindings*> seenBindings; + traceable_flat_hash_set<const Env*> seenEnvs; + traceable_flat_hash_set<const NixList*> seenLists; + traceable_flat_hash_set<const char*> seenStrings; + traceable_flat_hash_set<const Value*> seenValues; + + auto doString = [&](const char* s) -> size_t { + if (seenStrings.find(s) != seenStrings.end()) { + return 0; + } + seenStrings.insert(s); + return strlen(s) + 1; + }; + + std::function<size_t(const Value& v)> doValue; + std::function<size_t(const Env& v)> doEnv; + + doValue = [&](const Value& v) -> size_t { + if (seenValues.find(&v) != seenValues.end()) { + return 0; + } + seenValues.insert(&v); + + size_t sz = sizeof(Value); + + switch (v.type) { + case tString: + sz += doString(v.string.s); + if (v.string.context != nullptr) { + for (const char** p = v.string.context; *p != nullptr; ++p) { + sz += doString(*p); + } + } + break; + case tPath: + sz += doString(v.path); + break; + case tAttrs: + if (seenBindings.find(v.attrs.get()) == seenBindings.end()) { + seenBindings.insert(v.attrs.get()); + sz += sizeof(Bindings); + for (const auto& i : *v.attrs) { + sz += doValue(*i.second.value); + } + } + break; + case tList: + if (seenLists.find(v.list.get()) == seenLists.end()) { + seenLists.insert(v.list.get()); + sz += v.listSize() * sizeof(Value*); + for (const Value* v : *v.list) { + sz += doValue(*v); + } + } + break; + case tThunk: + sz += doEnv(*v.thunk.env); + break; + case tApp: + sz += doValue(*v.app.left); + sz += doValue(*v.app.right); + break; + case tLambda: + sz += doEnv(*v.lambda.env); + break; + case tPrimOpApp: + sz += doValue(*v.primOpApp.left); + sz += doValue(*v.primOpApp.right); + break; + default:; + } + + return sz; + }; + + doEnv = [&](const Env& env) -> size_t { + if (seenEnvs.find(&env) != seenEnvs.end()) { + return 0; + } + seenEnvs.insert(&env); + + size_t sz = sizeof(Env) + sizeof(Value*) * env.size; + + if (env.type != Env::HasWithExpr) { + for (const Value* v : env.values) { + if (v != nullptr) { + sz += doValue(*v); + } + } + } else { + // TODO(kanepyork): trace ExprWith? how important is this accounting? + } + + if (env.up != nullptr) { + sz += doEnv(*env.up); + } + + return sz; + }; + + return doValue(v); +} + +EvalSettings evalSettings; + +static GlobalConfig::Register r1(&evalSettings); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/eval.hh b/third_party/nix/src/libexpr/eval.hh new file mode 100644 index 0000000000..0352a89a2a --- /dev/null +++ b/third_party/nix/src/libexpr/eval.hh @@ -0,0 +1,365 @@ +#pragma once + +#include <map> +#include <optional> +#include <unordered_map> +#include <vector> + +#include "libexpr/attr-set.hh" +#include "libexpr/nixexpr.hh" +#include "libexpr/symbol-table.hh" +#include "libexpr/value.hh" +#include "libutil/config.hh" +#include "libutil/hash.hh" + +namespace nix { +namespace expr { + +// Initialise the Boehm GC once per program instance. This should be +// called in places that require the garbage collector. +void InitGC(); + +} // namespace expr + +class Store; +class EvalState; +enum RepairFlag : bool; + +typedef void (*PrimOpFun)(EvalState& state, const Pos& pos, Value** args, + Value& v); + +struct PrimOp { + PrimOpFun fun; + size_t arity; + Symbol name; + PrimOp(PrimOpFun fun, size_t arity, Symbol name) + : fun(fun), arity(arity), name(name) {} +}; + +struct Env { + Env(unsigned short size) : size(size) { values = std::vector<Value*>(size); } + + Env* up; + unsigned short size; // used by ‘valueSize’ + unsigned short prevWith : 14; // nr of levels up to next `with' environment + enum { Plain = 0, HasWithExpr, HasWithAttrs } type : 2; + std::vector<Value*> values; + Expr* withAttrsExpr = nullptr; +}; + +Value& mkString(Value& v, const std::string& s, + const PathSet& context = PathSet()); + +void copyContext(const Value& v, PathSet& context); + +/* Cache for calls to addToStore(); maps source paths to the store + paths. */ +using SrcToStore = std::map<Path, Path>; + +std::ostream& operator<<(std::ostream& str, const Value& v); + +using SearchPathElem = std::pair<std::string, std::string>; +using SearchPath = std::list<SearchPathElem>; + +using FileParseCache = std::map<Path, Expr*>; + +class EvalState { + public: + SymbolTable symbols; + + const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, + sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, + sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, sOutputHash, + sOutputHashAlgo, sOutputHashMode; + + // Symbol representing the path to the built-in 'derivation.nix' + // file, set during primops initialisation. + std::optional<Symbol> sDerivationNix; + + /* If set, force copying files to the Nix store even if they + already exist there. */ + RepairFlag repair; + + /* The allowed filesystem paths in restricted or pure evaluation + mode. */ + std::optional<PathSet> allowedPaths; + + const ref<Store> store; + + private: + SrcToStore srcToStore; + + /* A cache from path names to parse trees. */ + FileParseCache fileParseCache; + + /* A cache from path names to values. */ + using FileEvalCache = std::map<Path, Value>; + FileEvalCache fileEvalCache; + + SearchPath searchPath; + + std::map<std::string, std::pair<bool, std::string>> searchPathResolved; + + /* Cache used by checkSourcePath(). */ + std::unordered_map<Path, Path> resolvedPaths; + + public: + EvalState(const Strings& _searchPath, const ref<Store>& store); + ~EvalState(); + + void addToSearchPath(const std::string& s); + + SearchPath getSearchPath() { return searchPath; } + + Path checkSourcePath(const Path& path); + + void checkURI(const std::string& uri); + + /* When using a diverted store and 'path' is in the Nix store, map + 'path' to the diverted location (e.g. /nix/store/foo is mapped + to /home/alice/my-nix/nix/store/foo). However, this is only + done if the context is not empty, since otherwise we're + probably trying to read from the actual /nix/store. This is + intended to distinguish between import-from-derivation and + sources stored in the actual /nix/store. */ + Path toRealPath(const Path& path, const PathSet& context); + + /* Parse a Nix expression from the specified file. */ + Expr* parseExprFromFile(const Path& path); + Expr* parseExprFromFile(const Path& path, StaticEnv& staticEnv); + + /* Parse a Nix expression from the specified string. */ + Expr* parseExprFromString(const std::string& s, const Path& basePath, + StaticEnv& staticEnv); + Expr* parseExprFromString(const std::string& s, const Path& basePath); + + Expr* parseStdin(); + + /* Evaluate an expression read from the given file to normal + form. */ + void evalFile(const Path& path, Value& v); + + void resetFileCache(); + + /* Look up a file in the search path. */ + Path findFile(const std::string& path); + Path findFile(SearchPath& searchPath, const std::string& path, + const Pos& pos = noPos); + + /* If the specified search path element is a URI, download it. */ + std::pair<bool, std::string> resolveSearchPathElem( + const SearchPathElem& elem); + + /* Evaluate an expression to normal form, storing the result in + value `v'. */ + void eval(Expr* e, Value& v); + + /* Evaluation the expression, then verify that it has the expected + type. */ + inline bool evalBool(Env& env, Expr* e); + inline bool evalBool(Env& env, Expr* e, const Pos& pos); + inline void evalAttrs(Env& env, Expr* e, Value& v); + + /* If `v' is a thunk, enter it and overwrite `v' with the result + of the evaluation of the thunk. If `v' is a delayed function + application, call the function and overwrite `v' with the + result. Otherwise, this is a no-op. */ + inline void forceValue(Value& v, const Pos& pos = noPos); + + /* Force a value, then recursively force list elements and + attributes. */ + void forceValueDeep(Value& v); + + /* Force `v', and then verify that it has the expected type. */ + NixInt forceInt(Value& v, const Pos& pos); + NixFloat forceFloat(Value& v, const Pos& pos); + bool forceBool(Value& v, const Pos& pos); + inline void forceAttrs(Value& v); + inline void forceAttrs(Value& v, const Pos& pos); + inline void forceList(Value& v); + inline void forceList(Value& v, const Pos& pos); + void forceFunction(Value& v, const Pos& pos); // either lambda or primop + std::string forceString(Value& v, const Pos& pos = noPos); + std::string forceString(Value& v, PathSet& context, const Pos& pos = noPos); + std::string forceStringNoCtx(Value& v, const Pos& pos = noPos); + + /* Return true iff the value `v' denotes a derivation (i.e. a + set with attribute `type = "derivation"'). */ + bool isDerivation(Value& v); + + std::optional<std::string> tryAttrsToString(const Pos& pos, Value& v, + PathSet& context, + bool coerceMore = false, + bool copyToStore = true); + + /* String coercion. Converts strings, paths and derivations to a + string. If `coerceMore' is set, also converts nulls, integers, + booleans and lists to a string. If `copyToStore' is set, + referenced paths are copied to the Nix store as a side effect. */ + std::string coerceToString(const Pos& pos, Value& v, PathSet& context, + bool coerceMore = false, bool copyToStore = true); + + std::string copyPathToStore(PathSet& context, const Path& path); + + /* Path coercion. Converts strings, paths and derivations to a + path. The result is guaranteed to be a canonicalised, absolute + path. Nothing is copied to the store. */ + Path coerceToPath(const Pos& pos, Value& v, PathSet& context); + + public: + /* The base environment, containing the builtin functions and + values. */ + Env& baseEnv; + + /* The same, but used during parsing to resolve variables. */ + StaticEnv staticBaseEnv; // !!! should be private + + private: + unsigned int baseEnvDispl = 0; + + void createBaseEnv(); + + Value* addConstant(const std::string& name, Value& v); + + Value* addPrimOp(const std::string& name, size_t arity, PrimOpFun primOp); + + public: + Value& getBuiltin(const std::string& name); + + private: + inline Value* lookupVar(Env* env, const ExprVar& var, bool noEval); + + friend struct ExprVar; + friend struct ExprAttrs; + friend struct ExprLet; + + Expr* parse(const char* text, const Path& path, const Path& basePath, + StaticEnv& staticEnv); + + public: + /* Do a deep equality test between two values. That is, list + elements and attributes are compared recursively. */ + bool eqValues(Value& v1, Value& v2); + + bool isFunctor(Value& fun); + + void callFunction(Value& fun, Value& arg, Value& v, const Pos& pos); + void callPrimOp(Value& fun, Value& arg, Value& v, const Pos& pos); + + /* Automatically call a function for which each argument has a + default value or has a binding in the `args' map. 'args' need + not live past the end of the call. */ + void autoCallFunction(Bindings* args, Value& fun, Value& res); + + /* Allocation primitives. */ + Value* allocValue(); + Env& allocEnv(size_t size); + + Value* allocAttr(Value& vAttrs, const Symbol& name); + + // Create a list value from the specified vector. + void mkList(Value& v, std::shared_ptr<NixList> list); + + // Create a list value, allocating as many elements as specified in + // size. This is used for the many cases in this codebase where + // assignment happens into the preallocated list. + void mkList(Value& v, size_t size = 0); + + void mkAttrs(Value& v, size_t capacity); + void mkThunk_(Value& v, Expr* expr); + void mkPos(Value& v, Pos* pos); + + void concatLists(Value& v, const NixList& lists, const Pos& pos); + + /* Print statistics. */ + void printStats(); + + void realiseContext(const PathSet& context); + + /* File access tracing. */ + void TraceFileAccess(const Path& path); + void EnableFileAccessTracing(std::function<void(const Path&)> fn); + + private: + unsigned long nrEnvs = 0; + unsigned long nrValuesInEnvs = 0; + unsigned long nrValues = 0; + unsigned long nrListElems = 0; + unsigned long nrAttrsets = 0; + unsigned long nrAttrsInAttrsets = 0; + unsigned long nrOpUpdates = 0; + unsigned long nrOpUpdateValuesCopied = 0; + unsigned long nrListConcats = 0; + unsigned long nrPrimOpCalls = 0; + unsigned long nrFunctionCalls = 0; + + bool countCalls; + + std::function<void(const Path&)> file_access_trace_fn = nullptr; + Path last_traced_file = ""; + + using PrimOpCalls = std::map<Symbol, size_t>; + PrimOpCalls primOpCalls; + + using FunctionCalls = std::map<ExprLambda*, size_t>; + FunctionCalls functionCalls; + + void incrFunctionCall(ExprLambda* fun); + + using AttrSelects = std::map<Pos, size_t>; + AttrSelects attrSelects; + + friend struct ExprOpUpdate; + friend struct ExprOpConcatLists; + friend struct ExprSelect; + friend void prim_getAttr(EvalState& state, const Pos& pos, Value** args, + Value& v); +}; + +/* Return a string representing the type of the value `v'. */ +std::string showType(const Value& v); + +/* Decode a context string ‘!<name>!<path>’ into a pair <path, + name>. */ +std::pair<std::string, std::string> decodeContext(const std::string& s); + +/* If `path' refers to a directory, then append "/default.nix". */ +Path resolveExprPath(Path path); + +struct InvalidPathError : EvalError { + Path path; + InvalidPathError(const Path& path); +#ifdef EXCEPTION_NEEDS_THROW_SPEC + ~InvalidPathError() noexcept {}; +#endif +}; + +struct EvalSettings : Config { + Setting<bool> restrictEval{ + this, false, "restrict-eval", + "Whether to restrict file system access to paths in $NIX_PATH, " + "and network access to the URI prefixes listed in 'allowed-uris'."}; + + Setting<bool> pureEval{this, false, "pure-eval", + "Whether to restrict file system and network access " + "to files specified by cryptographic hash."}; + + Setting<bool> enableImportFromDerivation{ + this, true, "allow-import-from-derivation", + "Whether the evaluator allows importing the result of a derivation."}; + + Setting<Strings> allowedUris{ + this, + {}, + "allowed-uris", + "Prefixes of URIs that builtin functions such as fetchurl and fetchGit " + "are allowed to fetch."}; + + Setting<bool> traceFunctionCalls{this, false, "trace-function-calls", + "Emit log messages for each function entry " + "and exit at the 'vomit' log level (-vvvv)"}; +}; + +extern EvalSettings evalSettings; + +} // namespace nix diff --git a/third_party/nix/src/libexpr/function-trace.cc b/third_party/nix/src/libexpr/function-trace.cc new file mode 100644 index 0000000000..b1b856965c --- /dev/null +++ b/third_party/nix/src/libexpr/function-trace.cc @@ -0,0 +1,19 @@ +#include "libexpr/function-trace.hh" + +#include <glog/logging.h> + +namespace nix { + +FunctionCallTrace::FunctionCallTrace(const Pos& pos) : pos(pos) { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); + LOG(INFO) << "function-trace entered " << pos << " at " << ns.count(); +} + +FunctionCallTrace::~FunctionCallTrace() { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); + LOG(INFO) << "function-trace exited " << pos << " at " << ns.count(); +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/function-trace.hh b/third_party/nix/src/libexpr/function-trace.hh new file mode 100644 index 0000000000..6b810159b8 --- /dev/null +++ b/third_party/nix/src/libexpr/function-trace.hh @@ -0,0 +1,14 @@ +#pragma once + +#include <chrono> + +#include "libexpr/eval.hh" + +namespace nix { + +struct FunctionCallTrace { + const Pos& pos; + FunctionCallTrace(const Pos& pos); + ~FunctionCallTrace(); +}; +} // namespace nix diff --git a/third_party/nix/src/libexpr/get-drvs.cc b/third_party/nix/src/libexpr/get-drvs.cc new file mode 100644 index 0000000000..164c1e54f3 --- /dev/null +++ b/third_party/nix/src/libexpr/get-drvs.cc @@ -0,0 +1,446 @@ +#include "libexpr/get-drvs.hh" + +#include <cstring> +#include <regex> +#include <utility> + +#include <absl/container/flat_hash_set.h> +#include <absl/strings/numbers.h> +#include <glog/logging.h> + +#include "libexpr/eval-inline.hh" +#include "libstore/derivations.hh" +#include "libutil/util.hh" + +namespace nix { + +DrvInfo::DrvInfo(EvalState& state, std::string attrPath, + std::shared_ptr<Bindings> attrs) + : state(&state), attrs(attrs), attrPath(std::move(attrPath)) {} + +DrvInfo::DrvInfo(EvalState& state, const ref<Store>& store, + const std::string& drvPathWithOutputs) + : state(&state), attrPath("") { + auto spec = parseDrvPathWithOutputs(drvPathWithOutputs); + + drvPath = spec.first; + + auto drv = store->derivationFromPath(drvPath); + + name = storePathToName(drvPath); + + if (spec.second.size() > 1) { + throw Error( + "building more than one derivation output is not supported, in '%s'", + drvPathWithOutputs); + } + + outputName = spec.second.empty() ? get(drv.env, "outputName", "out") + : *spec.second.begin(); + + auto i = drv.outputs.find(outputName); + if (i == drv.outputs.end()) { + throw Error("derivation '%s' does not have output '%s'", drvPath, + outputName); + } + + outPath = i->second.path; +} + +std::string DrvInfo::queryName() const { + if (name.empty() && (attrs != nullptr)) { + auto i = attrs->find(state->sName); + if (i == attrs->end()) { + throw TypeError("derivation name missing"); + } + name = state->forceStringNoCtx(*i->second.value); + } + return name; +} + +std::string DrvInfo::querySystem() const { + if (system.empty() && (attrs != nullptr)) { + auto i = attrs->find(state->sSystem); + system = i == attrs->end() + ? "unknown" + : state->forceStringNoCtx(*i->second.value, *i->second.pos); + } + return system; +} + +std::string DrvInfo::queryDrvPath() const { + if (drvPath.empty() && (attrs != nullptr)) { + Bindings::iterator i = attrs->find(state->sDrvPath); + PathSet context; + drvPath = i != attrs->end() ? state->coerceToPath(*i->second.pos, + *i->second.value, context) + : ""; + } + return drvPath; +} + +std::string DrvInfo::queryOutPath() const { + if (outPath.empty() && (attrs != nullptr)) { + Bindings::iterator i = attrs->find(state->sOutPath); + PathSet context; + outPath = i != attrs->end() ? state->coerceToPath(*i->second.pos, + *i->second.value, context) + : ""; + } + return outPath; +} + +DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) { + if (outputs.empty()) { + /* Get the ‘outputs’ list. */ + Bindings::iterator i; + if ((attrs != nullptr) && + (i = attrs->find(state->sOutputs)) != attrs->end()) { + state->forceList(*i->second.value, *i->second.pos); + + /* For each output... */ + for (unsigned int j = 0; j < i->second.value->listSize(); ++j) { + /* Evaluate the corresponding set. */ + std::string name = state->forceStringNoCtx(*(*i->second.value->list)[j], + *i->second.pos); + Bindings::iterator out = attrs->find(state->symbols.Create(name)); + if (out == attrs->end()) { + continue; // FIXME: throw error? + } + state->forceAttrs(*out->second.value); + + /* And evaluate its ‘outPath’ attribute. */ + Bindings::iterator outPath = + out->second.value->attrs->find(state->sOutPath); + if (outPath == out->second.value->attrs->end()) { + continue; // FIXME: throw error? + } + PathSet context; + outputs[name] = state->coerceToPath(*outPath->second.pos, + *outPath->second.value, context); + } + } else { + outputs["out"] = queryOutPath(); + } + } + if (!onlyOutputsToInstall || (attrs == nullptr)) { + return outputs; + } + + /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ + const Value* outTI = queryMeta("outputsToInstall"); + if (outTI == nullptr) { + return outputs; + } + const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); + /* ^ this shows during `nix-env -i` right under the bad derivation */ + if (!outTI->isList()) { + throw errMsg; + } + Outputs result; + + for (Value* i : *outTI->list) { + if (i->type != tString) { + throw errMsg; + } + auto out = outputs.find(i->string.s); + if (out == outputs.end()) { + throw errMsg; + } + result.insert(*out); + } + return result; +} + +std::string DrvInfo::queryOutputName() const { + if (outputName.empty() && (attrs != nullptr)) { + Bindings::iterator i = attrs->find(state->sOutputName); + outputName = + i != attrs->end() ? state->forceStringNoCtx(*i->second.value) : ""; + } + return outputName; +} + +Bindings* DrvInfo::getMeta() { + if (meta != nullptr) { + return meta.get(); + } + if (attrs == nullptr) { + return nullptr; + } + Bindings::iterator a = attrs->find(state->sMeta); + if (a == attrs->end()) { + return nullptr; + } + state->forceAttrs(*a->second.value, *a->second.pos); + meta = a->second.value->attrs; + return meta.get(); +} + +StringSet DrvInfo::queryMetaNames() { + StringSet res; + if (getMeta() == nullptr) { + return res; + } + for (auto& i : *meta) { + res.insert(i.second.name); + } + return res; +} + +bool DrvInfo::checkMeta(Value& v) { + state->forceValue(v); + if (v.isList()) { + for (unsigned int n = 0; n < v.listSize(); ++n) { + if (!checkMeta(*(*v.list)[n])) { + return false; + } + } + return true; + } + if (v.type == tAttrs) { + Bindings::iterator i = v.attrs->find(state->sOutPath); + if (i != v.attrs->end()) { + return false; + } + for (auto& i : *v.attrs) { + if (!checkMeta(*i.second.value)) { + return false; + } + } + return true; + } else { + return v.type == tInt || v.type == tBool || v.type == tString || + v.type == tFloat; + } +} + +Value* DrvInfo::queryMeta(const std::string& name) { + if (getMeta() == nullptr) { + return nullptr; + } + Bindings::iterator a = meta->find(state->symbols.Create(name)); + if (a == meta->end() || !checkMeta(*a->second.value)) { + return nullptr; + } + return a->second.value; +} + +std::string DrvInfo::queryMetaString(const std::string& name) { + Value* v = queryMeta(name); + if ((v == nullptr) || v->type != tString) { + return ""; + } + return v->string.s; +} + +NixInt DrvInfo::queryMetaInt(const std::string& name, NixInt def) { + Value* v = queryMeta(name); + if (v == nullptr) { + return def; + } + if (v->type == tInt) { + return v->integer; + } + if (v->type == tString) { + /* Backwards compatibility with before we had support for + integer meta fields. */ + NixInt n; + if (absl::SimpleAtoi(v->string.s, &n)) { + return n; + } + } + return def; +} + +NixFloat DrvInfo::queryMetaFloat(const std::string& name, NixFloat def) { + Value* v = queryMeta(name); + if (v == nullptr) { + return def; + } + if (v->type == tFloat) { + return v->fpoint; + } + if (v->type == tString) { + /* Backwards compatibility with before we had support for + float meta fields. */ + NixFloat n; + if (string2Float(v->string.s, n)) { + return n; + } + } + return def; +} + +bool DrvInfo::queryMetaBool(const std::string& name, bool def) { + Value* v = queryMeta(name); + if (v == nullptr) { + return def; + } + if (v->type == tBool) { + return v->boolean; + } + if (v->type == tString) { + /* Backwards compatibility with before we had support for + Boolean meta fields. */ + if (strcmp(v->string.s, "true") == 0) { + return true; + } + if (strcmp(v->string.s, "false") == 0) { + return false; + } + } + return def; +} + +void DrvInfo::setMeta(const std::string& name, Value* v) { + std::shared_ptr<Bindings> old = meta; + meta = std::shared_ptr<Bindings>(Bindings::New(old->size() + 1).release()); + Symbol sym = state->symbols.Create(name); + if (old != nullptr) { + for (auto i : *old) { + if (i.second.name != sym) { + meta->push_back(i.second); + } + } + } + if (v != nullptr) { + meta->push_back(Attr(sym, v)); + } +} + +/* Cache for already considered attrsets. */ +using Done = absl::flat_hash_set<std::shared_ptr<Bindings>>; + +/* Evaluate value `v'. If it evaluates to a set of type `derivation', + then put information about it in `drvs' (unless it's already in `done'). + The result boolean indicates whether it makes sense + for the caller to recursively search for derivations in `v'. */ +static bool getDerivation(EvalState& state, Value& v, + const std::string& attrPath, DrvInfos& drvs, + Done& done, bool ignoreAssertionFailures) { + try { + state.forceValue(v); + if (!state.isDerivation(v)) { + return true; + } + + /* Remove spurious duplicates (e.g., a set like `rec { x = + derivation {...}; y = x;}'. */ + if (done.find(v.attrs) != done.end()) { + return false; + } + done.insert(v.attrs); + + DrvInfo drv(state, attrPath, v.attrs); + + drv.queryName(); + + drvs.push_back(drv); + + return false; + + } catch (AssertionError& e) { + if (ignoreAssertionFailures) { + return false; + } + throw; + } +} + +std::optional<DrvInfo> getDerivation(EvalState& state, Value& v, + bool ignoreAssertionFailures) { + Done done; + DrvInfos drvs; + getDerivation(state, v, "", drvs, done, ignoreAssertionFailures); + if (drvs.size() != 1) { + return {}; + } + return std::move(drvs.front()); +} + +static std::string addToPath(const std::string& s1, const std::string& s2) { + return s1.empty() ? s2 : s1 + "." + s2; +} + +static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*"); + +static void getDerivations(EvalState& state, Value& vIn, + const std::string& pathPrefix, Bindings* autoArgs, + DrvInfos& drvs, Done& done, + bool ignoreAssertionFailures) { + Value v; + state.autoCallFunction(autoArgs, vIn, v); + + /* Process the expression. */ + if (!getDerivation(state, v, pathPrefix, drvs, done, + ignoreAssertionFailures)) { + ; + + } else if (v.type == tAttrs) { + /* !!! undocumented hackery to support combining channels in + nix-env.cc. */ + bool combineChannels = + v.attrs->find(state.symbols.Create("_combineChannels")) != + v.attrs->end(); + + /* Consider the attributes in sorted order to get more + deterministic behaviour in nix-env operations (e.g. when + there are names clashes between derivations, the derivation + bound to the attribute with the "lower" name should take + precedence). */ + for (auto& [_, i] : *v.attrs) { + DLOG(INFO) << "evaluating attribute '" << i.name << "'"; + if (!std::regex_match(std::string(i.name), attrRegex)) { + continue; + } + std::string pathPrefix2 = addToPath(pathPrefix, i.name); + if (combineChannels) { + getDerivations(state, *i.value, pathPrefix2, autoArgs, drvs, done, + ignoreAssertionFailures); + } else if (getDerivation(state, *i.value, pathPrefix2, drvs, done, + ignoreAssertionFailures)) { + /* If the value of this attribute is itself a set, + should we recurse into it? => Only if it has a + `recurseForDerivations = true' attribute. */ + if (i.value->type == tAttrs) { + Bindings::iterator j = i.value->attrs->find( + state.symbols.Create("recurseForDerivations")); + if (j != i.value->attrs->end() && + state.forceBool(*j->second.value, *j->second.pos)) { + getDerivations(state, *i.value, pathPrefix2, autoArgs, drvs, done, + ignoreAssertionFailures); + } + } + } + } + } + + else if (v.isList()) { + for (unsigned int n = 0; n < v.listSize(); ++n) { + std::string pathPrefix2 = + addToPath(pathPrefix, (format("%1%") % n).str()); + if (getDerivation(state, *(*v.list)[n], pathPrefix2, drvs, done, + ignoreAssertionFailures)) { + getDerivations(state, *(*v.list)[n], pathPrefix2, autoArgs, drvs, done, + ignoreAssertionFailures); + } + } + } + + else { + throw TypeError( + "expression does not evaluate to a derivation (or a set or list of " + "those)"); + } +} + +void getDerivations(EvalState& state, Value& v, const std::string& pathPrefix, + Bindings* autoArgs, DrvInfos& drvs, + bool ignoreAssertionFailures) { + Done done; + getDerivations(state, v, pathPrefix, autoArgs, drvs, done, + ignoreAssertionFailures); +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/get-drvs.hh b/third_party/nix/src/libexpr/get-drvs.hh new file mode 100644 index 0000000000..3de266d0c0 --- /dev/null +++ b/third_party/nix/src/libexpr/get-drvs.hh @@ -0,0 +1,83 @@ +#pragma once + +#include <map> +#include <string> + +#include "libexpr/eval.hh" + +namespace nix { + +struct DrvInfo { + public: + typedef std::map<std::string, Path> Outputs; + + private: + EvalState* state; + + mutable std::string name; + mutable std::string system; + mutable std::string drvPath; + mutable std::string outPath; + mutable std::string outputName; + Outputs outputs; + + bool failed = false; // set if we get an AssertionError + + std::shared_ptr<Bindings> attrs = nullptr; + std::shared_ptr<Bindings> meta = nullptr; + + Bindings* getMeta(); + + bool checkMeta(Value& v); + + public: + std::string attrPath; /* path towards the derivation */ + + DrvInfo(EvalState& state) : state(&state){}; + DrvInfo(EvalState& state, std::string attrPath, + std::shared_ptr<Bindings> attrs); + DrvInfo(EvalState& state, const ref<Store>& store, + const std::string& drvPathWithOutputs); + + std::string queryName() const; + std::string querySystem() const; + std::string queryDrvPath() const; + std::string queryOutPath() const; + std::string queryOutputName() const; + /** Return the list of outputs. The "outputs to install" are determined by + * `meta.outputsToInstall`. */ + Outputs queryOutputs(bool onlyOutputsToInstall = false); + + StringSet queryMetaNames(); + Value* queryMeta(const std::string& name); + std::string queryMetaString(const std::string& name); + NixInt queryMetaInt(const std::string& name, NixInt def); + NixFloat queryMetaFloat(const std::string& name, NixFloat def); + bool queryMetaBool(const std::string& name, bool def); + void setMeta(const std::string& name, Value* v); + + /* + MetaInfo queryMetaInfo(EvalState & state) const; + MetaValue queryMetaInfo(EvalState & state, const std::string & name) const; + */ + + void setName(const std::string& s) { name = s; } + void setDrvPath(const std::string& s) { drvPath = s; } + void setOutPath(const std::string& s) { outPath = s; } + + void setFailed() { failed = true; }; + bool hasFailed() { return failed; }; +}; + +using DrvInfos = std::list<DrvInfo>; + +/* If value `v' denotes a derivation, return a DrvInfo object + describing it. Otherwise return nothing. */ +std::optional<DrvInfo> getDerivation(EvalState& state, Value& v, + bool ignoreAssertionFailures); + +void getDerivations(EvalState& state, Value& v, const std::string& pathPrefix, + Bindings* autoArgs, DrvInfos& drvs, + bool ignoreAssertionFailures); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/json-to-value.cc b/third_party/nix/src/libexpr/json-to-value.cc new file mode 100644 index 0000000000..043f8c64cd --- /dev/null +++ b/third_party/nix/src/libexpr/json-to-value.cc @@ -0,0 +1,152 @@ +#include "libexpr/json-to-value.hh" + +#include <nlohmann/json.hpp> +#include <variant> +#include <vector> + +#include "libexpr/value.hh" + +using json = nlohmann::json; + +namespace nix { + +// for more information, refer to +// https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp +class JSONSax : nlohmann::json_sax<json> { + class JSONState { + protected: + std::unique_ptr<JSONState> parent; + std::shared_ptr<Value*> v; + + public: + virtual std::unique_ptr<JSONState> resolve(EvalState&) { + throw std::logic_error("tried to close toplevel json parser state"); + } + explicit JSONState(std::unique_ptr<JSONState>&& p) : parent(std::move(p)) {} + explicit JSONState(Value* v) : v(allocRootValue(v)) {} + JSONState(JSONState& p) = delete; + Value& value(EvalState& state) { + if (!v) v = allocRootValue(state.allocValue()); + return **v; + } + virtual ~JSONState() {} + virtual void add() {} + }; + + class JSONObjectState : public JSONState { + using JSONState::JSONState; + ValueMap attrs = ValueMap(); + std::unique_ptr<JSONState> resolve(EvalState& state) override { + Value& v = parent->value(state); + state.mkAttrs(v, attrs.size()); + for (auto& i : attrs) v.attrs->push_back(Attr(i.first, i.second)); + return std::move(parent); + } + void add() override { v = nullptr; }; + + public: + void key(string_t& name, EvalState& state) { + attrs[state.symbols.Create(name)] = &value(state); + } + }; + + class JSONListState : public JSONState { + using JSONState::JSONState; + std::vector<Value*> values; + std::unique_ptr<JSONState> resolve(EvalState& state) override { + Value& v = parent->value(state); + state.mkList(v, values.size()); + for (size_t n = 0; n < values.size(); ++n) { + (*v.list)[n] = values[n]; + } + return std::move(parent); + } + void add() override { + values.push_back(*v); + v = nullptr; + }; + + public: + JSONListState(std::unique_ptr<JSONState>&& p, std::size_t reserve) + : JSONState(std::move(p)) { + values.reserve(reserve); + } + }; + + EvalState& state; + std::unique_ptr<JSONState> rs; + + template <typename T, typename... Args> + inline bool handle_value(T f, Args... args) { + f(rs->value(state), args...); + rs->add(); + return true; + } + + public: + JSONSax(EvalState& state, Value& v) : state(state), rs(new JSONState(&v)){}; + + bool null() override { return handle_value(mkNull); } + + bool boolean(bool val) override { return handle_value(mkBool, val); } + + bool number_integer(number_integer_t val) override { + return handle_value(mkInt, val); + } + + bool number_unsigned(number_unsigned_t val) override { + return handle_value(mkInt, val); + } + + bool number_float(number_float_t val, const string_t&) override { + return handle_value(mkFloat, val); + } + + bool string(string_t& val) override { + return handle_value<void(Value&, const char*)>(mkString, val.c_str()); + } + +#if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8 + bool binary(binary_t&) { + // This function ought to be unreachable + assert(false); + return true; + } +#endif + + bool start_object(std::size_t) override { + rs = std::make_unique<JSONObjectState>(std::move(rs)); + return true; + } + + bool key(string_t& name) override { + dynamic_cast<JSONObjectState*>(rs.get())->key(name, state); + return true; + } + + bool end_object() override { + rs = rs->resolve(state); + rs->add(); + return true; + } + + bool end_array() override { return end_object(); } + + bool start_array(size_t len) override { + rs = std::make_unique<JSONListState>( + std::move(rs), len != std::numeric_limits<size_t>::max() ? len : 128); + return true; + } + + bool parse_error(std::size_t, const std::string&, + const nlohmann::detail::exception& ex) override { + throw JSONParseError(ex.what()); + } +}; + +void parseJSON(EvalState& state, const std::string& s_, Value& v) { + JSONSax parser(state, v); + bool res = json::sax_parse(s_, &parser); + if (!res) throw JSONParseError("Invalid JSON Value"); +} +} // namespace nix diff --git a/third_party/nix/src/libexpr/json-to-value.hh b/third_party/nix/src/libexpr/json-to-value.hh new file mode 100644 index 0000000000..7f258f2137 --- /dev/null +++ b/third_party/nix/src/libexpr/json-to-value.hh @@ -0,0 +1,13 @@ +#pragma once + +#include <string> + +#include "libexpr/eval.hh" + +namespace nix { + +MakeError(JSONParseError, EvalError); + +void parseJSON(EvalState& state, const std::string& s, Value& v); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/lexer.l b/third_party/nix/src/libexpr/lexer.l new file mode 100644 index 0000000000..d5b8a45936 --- /dev/null +++ b/third_party/nix/src/libexpr/lexer.l @@ -0,0 +1,193 @@ +%option reentrant bison-bridge bison-locations +%option noyywrap +%option never-interactive +%option stack +%option nodefault +%option nounput noyy_top_state + + +%s DEFAULT +%x STRING +%x IND_STRING + + +%{ +#include <boost/lexical_cast.hpp> + +#include "generated/parser-tab.hh" +#include "libexpr/nixexpr.hh" +#include "libexpr/parser.hh" + +using namespace nix; + +namespace nix { + +static void initLoc(YYLTYPE* loc) { + loc->first_line = loc->last_line = 1; + loc->first_column = loc->last_column = 1; +} + +static void adjustLoc(YYLTYPE* loc, const char* s, size_t len) { + loc->first_line = loc->last_line; + loc->first_column = loc->last_column; + + while (len--) { + switch (*s++) { + case '\r': + if (*s == '\n') /* cr/lf */ + s++; + /* fall through */ + case '\n': + ++loc->last_line; + loc->last_column = 1; + break; + default: + ++loc->last_column; + } + } +} + +} + +#define YY_USER_INIT initLoc(yylloc) +#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng); + +#define PUSH_STATE(state) yy_push_state(state, yyscanner) +#define POP_STATE() yy_pop_state(yyscanner) + +%} + + +ANY .|\n +ID [a-zA-Z\_][a-zA-Z0-9\_\'\-]* +INT [0-9]+ +FLOAT (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)? +PATH [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+\/? +HPATH \~(\/[a-zA-Z0-9\.\_\-\+]+)+\/? +SPATH \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\> +URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ + + +%% + + +if { return IF; } +then { return THEN; } +else { return ELSE; } +assert { return ASSERT; } +with { return WITH; } +let { return LET; } +in { return IN; } +rec { return REC; } +inherit { return INHERIT; } +or { return OR_KW; } +\.\.\. { return ELLIPSIS; } + +\=\= { return EQ; } +\!\= { return NEQ; } +\<\= { return LEQ; } +\>\= { return GEQ; } +\&\& { return AND; } +\|\| { return OR; } +\-\> { return IMPL; } +\/\/ { return UPDATE; } +\+\+ { return CONCAT; } + +{ID} { yylval->id = strdup(yytext); return ID; } +{INT} { errno = 0; + try { + yylval->n = boost::lexical_cast<int64_t>(yytext); + } catch (const boost::bad_lexical_cast &) { + throw ParseError(format("invalid integer '%1%'") % yytext); + } + return INT; + } +{FLOAT} { errno = 0; + yylval->nf = strtod(yytext, 0); + if (errno != 0) + throw ParseError(format("invalid float '%1%'") % yytext); + return FLOAT; + } + +\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } + +\} { /* State INITIAL only exists at the bottom of the stack and is + used as a marker. DEFAULT replaces it everywhere else. + Popping when in INITIAL state causes an empty stack exception, + so don't */ + if (YYSTATE != INITIAL) + POP_STATE(); + return '}'; + } +\{ { PUSH_STATE(DEFAULT); return '{'; } + +\" { PUSH_STATE(STRING); return '"'; } +<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" | +<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ { + /* It is impossible to match strings ending with '$' with one + regex because trailing contexts are only valid at the end + of a rule. (A sane but undocumented limitation.) */ + yylval->e = unescapeStr(data->symbols, yytext, yyleng); + return STR; + } +<STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } +<STRING>\" { POP_STATE(); return '"'; } +<STRING>\$|\\|\$\\ { + /* This can only occur when we reach EOF, otherwise the above + (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered. + This is technically invalid, but we leave the problem to the + parser who fails with exact location. */ + return STR; + } + +\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } +<IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ { + yylval->e = new ExprIndStr(yytext); + return IND_STR; + } +<IND_STRING>\'\'\$ | +<IND_STRING>\$ { + yylval->e = new ExprIndStr("$"); + return IND_STR; + } +<IND_STRING>\'\'\' { + yylval->e = new ExprIndStr("''"); + return IND_STR; + } +<IND_STRING>\'\'\\{ANY} { + yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2); + return IND_STR; + } +<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } +<IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; } +<IND_STRING>\' { + yylval->e = new ExprIndStr("'"); + return IND_STR; + } + + +{PATH} { if (yytext[yyleng-1] == '/') + throw ParseError("path '%s' has a trailing slash", yytext); + yylval->path = strdup(yytext); + return PATH; + } +{HPATH} { if (yytext[yyleng-1] == '/') + throw ParseError("path '%s' has a trailing slash", yytext); + yylval->path = strdup(yytext); + return HPATH; + } +{SPATH} { yylval->path = strdup(yytext); return SPATH; } +{URI} { yylval->uri = strdup(yytext); return URI; } + +[ \t\r\n]+ /* eat up whitespace */ +\#[^\r\n]* /* single-line comments */ +\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ + +{ANY} { + /* Don't return a negative number, as this will cause + Bison to stop parsing without an error. */ + return (unsigned char) yytext[0]; + } + +%% + diff --git a/third_party/nix/src/libexpr/names.cc b/third_party/nix/src/libexpr/names.cc new file mode 100644 index 0000000000..1e9c2f2f4a --- /dev/null +++ b/third_party/nix/src/libexpr/names.cc @@ -0,0 +1,121 @@ +#include "libexpr/names.hh" + +#include <memory> + +#include <absl/strings/numbers.h> + +#include "libutil/util.hh" + +namespace nix { + +DrvName::DrvName() { name = ""; } + +/* Parse a derivation name. The `name' part of a derivation name is + everything up to but not including the first dash *not* followed by + a letter. The `version' part is the rest (excluding the separating + dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd', + '2.0.48'). */ +DrvName::DrvName(const std::string& s) : hits(0) { + name = fullName = s; + for (unsigned int i = 0; i < s.size(); ++i) { + /* !!! isalpha/isdigit are affected by the locale. */ + if (s[i] == '-' && i + 1 < s.size() && (isalpha(s[i + 1]) == 0)) { + name = std::string(s, 0, i); + version = std::string(s, i + 1); + break; + } + } +} + +bool DrvName::matches(DrvName& n) { + if (name != "*") { + if (!regex) { + regex = std::make_unique<std::regex>(name, std::regex::extended); + } + if (!std::regex_match(n.name, *regex)) { + return false; + } + } + return !(!version.empty() && version != n.version); +} + +std::string nextComponent(std::string::const_iterator& p, + const std::string::const_iterator end) { + /* Skip any dots and dashes (component separators). */ + while (p != end && (*p == '.' || *p == '-')) { + ++p; + } + + if (p == end) { + return ""; + } + + /* If the first character is a digit, consume the longest sequence + of digits. Otherwise, consume the longest sequence of + non-digit, non-separator characters. */ + std::string s; + if (isdigit(*p) != 0) { + while (p != end && (isdigit(*p) != 0)) { + s += *p++; + } + } else { + while (p != end && ((isdigit(*p) == 0) && *p != '.' && *p != '-')) { + s += *p++; + } + } + + return s; +} + +static bool componentsLT(const std::string& c1, const std::string& c2) { + int n1; + int n2; + bool c1Num = absl::SimpleAtoi(c1, &n1); + bool c2Num = absl::SimpleAtoi(c2, &n2); + + if (c1Num && c2Num) { + return n1 < n2; + } + if (c1.empty() && c2Num) { + return true; + } else if (c1 == "pre" && c2 != "pre") { + return true; + } else if (c2 == "pre") { + return false; + /* Assume that `2.3a' < `2.3.1'. */ + } else if (c2Num) { + return true; + } else if (c1Num) { + return false; + } else { + return c1 < c2; + } +} + +int compareVersions(const std::string& v1, const std::string& v2) { + std::string::const_iterator p1 = v1.begin(); + std::string::const_iterator p2 = v2.begin(); + + while (p1 != v1.end() || p2 != v2.end()) { + std::string c1 = nextComponent(p1, v1.end()); + std::string c2 = nextComponent(p2, v2.end()); + if (componentsLT(c1, c2)) { + return -1; + } + if (componentsLT(c2, c1)) { + return 1; + } + } + + return 0; +} + +DrvNames drvNamesFromArgs(const Strings& opArgs) { + DrvNames result; + for (auto& i : opArgs) { + result.push_back(DrvName(i)); + } + return result; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/names.hh b/third_party/nix/src/libexpr/names.hh new file mode 100644 index 0000000000..061388d517 --- /dev/null +++ b/third_party/nix/src/libexpr/names.hh @@ -0,0 +1,31 @@ +#pragma once + +#include <memory> +#include <regex> + +#include "libutil/types.hh" + +namespace nix { + +struct DrvName { + std::string fullName; + std::string name; + std::string version; + unsigned int hits; + + DrvName(); + DrvName(const std::string& s); + bool matches(DrvName& n); + + private: + std::unique_ptr<std::regex> regex; +}; + +typedef std::list<DrvName> DrvNames; + +std::string nextComponent(std::string::const_iterator& p, + const std::string::const_iterator end); +int compareVersions(const std::string& v1, const std::string& v2); +DrvNames drvNamesFromArgs(const Strings& opArgs); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/nix-expr.pc.in b/third_party/nix/src/libexpr/nix-expr.pc.in new file mode 100644 index 0000000000..99b0ae2c68 --- /dev/null +++ b/third_party/nix/src/libexpr/nix-expr.pc.in @@ -0,0 +1,10 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +libdir=@CMAKE_INSTALL_FULL_LIBDIR@ +includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Requires: nix-store bdw-gc +Libs: -L${libdir} -lnixexpr +Cflags: -I${includedir}/nix diff --git a/third_party/nix/src/libexpr/nixexpr.cc b/third_party/nix/src/libexpr/nixexpr.cc new file mode 100644 index 0000000000..391f068205 --- /dev/null +++ b/third_party/nix/src/libexpr/nixexpr.cc @@ -0,0 +1,414 @@ +#include "libexpr/nixexpr.hh" + +#include <cstdlib> +#include <variant> + +#include "libstore/derivations.hh" +#include "libutil/util.hh" +#include "libutil/visitor.hh" + +namespace nix { + +/* Displaying abstract syntax trees. */ + +std::ostream& operator<<(std::ostream& str, const Expr& e) { + e.show(str); + return str; +} + +static void showString(std::ostream& str, const std::string& s) { + str << '"'; + for (auto c : std::string(s)) { + if (c == '"' || c == '\\' || c == '$') { + str << "\\" << c; + } else if (c == '\n') { + str << "\\n"; + } else if (c == '\r') { + str << "\\r"; + } else if (c == '\t') { + str << "\\t"; + } else { + str << c; + } + } + str << '"'; +} + +static void showId(std::ostream& str, const std::string& s) { + if (s.empty()) { + str << "\"\""; + } else if (s == "if") { // FIXME: handle other keywords + str << '"' << s << '"'; + } else { + char c = s[0]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { + showString(str, s); + return; + } + for (auto c : s) { + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_' || c == '\'' || c == '-')) { + showString(str, s); + return; + } + } + str << s; + } +} + +std::ostream& operator<<(std::ostream& str, const Symbol& sym) { + showId(str, *sym.s); + return str; +} + +void Expr::show(std::ostream& str) const { abort(); } + +void ExprInt::show(std::ostream& str) const { str << n; } + +void ExprFloat::show(std::ostream& str) const { str << nf; } + +void ExprString::show(std::ostream& str) const { showString(str, s); } + +void ExprPath::show(std::ostream& str) const { str << s; } + +void ExprVar::show(std::ostream& str) const { str << name; } + +void ExprSelect::show(std::ostream& str) const { + str << "(" << *e << ")." << showAttrPath(attrPath); + if (def != nullptr) { + str << " or (" << *def << ")"; + } +} + +void ExprOpHasAttr::show(std::ostream& str) const { + str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")"; +} + +void ExprAttrs::show(std::ostream& str) const { + if (recursive) { + str << "rec "; + } + str << "{ "; + for (auto& i : attrs) { + if (i.second.inherited) { + str << "inherit " << i.first << " " + << "; "; + } else { + str << i.first << " = " << *i.second.e << "; "; + } + } + for (auto& i : dynamicAttrs) { + str << "\"${" << *i.nameExpr << "}\" = " << *i.valueExpr << "; "; + } + str << "}"; +} + +void ExprList::show(std::ostream& str) const { + str << "[ "; + for (auto& i : elems) { + str << "(" << *i << ") "; + } + str << "]"; +} + +void ExprLambda::show(std::ostream& str) const { + str << "("; + if (matchAttrs) { + str << "{ "; + bool first = true; + for (auto& i : formals->formals) { + if (first) { + first = false; + } else { + str << ", "; + } + str << i.name; + if (i.def != nullptr) { + str << " ? " << *i.def; + } + } + if (formals->ellipsis) { + if (!first) { + str << ", "; + } + str << "..."; + } + str << " }"; + if (!arg.empty()) { + str << " @ "; + } + } + if (!arg.empty()) { + str << arg; + } + str << ": " << *body << ")"; +} + +void ExprLet::show(std::ostream& str) const { + str << "(let "; + for (auto& i : attrs->attrs) { + if (i.second.inherited) { + str << "inherit " << i.first << "; "; + } else { + str << i.first << " = " << *i.second.e << "; "; + } + } + str << "in " << *body << ")"; +} + +void ExprWith::show(std::ostream& str) const { + str << "(with " << *attrs << "; " << *body << ")"; +} + +void ExprIf::show(std::ostream& str) const { + str << "(if " << *cond << " then " << *then << " else " << *else_ << ")"; +} + +void ExprAssert::show(std::ostream& str) const { + str << "assert " << *cond << "; " << *body; +} + +void ExprOpNot::show(std::ostream& str) const { str << "(! " << *e << ")"; } + +void ExprConcatStrings::show(std::ostream& str) const { + bool first = true; + str << "("; + for (auto& i : *es) { + if (first) { + first = false; + } else { + str << " + "; + } + str << *i; + } + str << ")"; +} + +void ExprPos::show(std::ostream& str) const { str << "__curPos"; } + +std::ostream& operator<<(std::ostream& str, const Pos& pos) { + if (!pos || !pos.file.has_value()) { + str << "undefined position"; + } else { + str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % + std::string(pos.file.value()) % pos.line % pos.column) + .str(); + } + return str; +} + +std::string showAttrPath(const AttrPath& attrPath) { + std::ostringstream out; + bool first = true; + for (auto& attr : attrPath) { + if (!first) { + out << '.'; + } else { + first = false; + } + + std::visit(util::overloaded{ + [&](const Symbol& sym) { out << sym; }, + [&](const Expr* expr) { out << "\"${" << *expr << "}\""; }}, + attr); + } + return out.str(); +} + +Pos noPos; + +/* Computing levels/displacements for variables. */ + +void Expr::bindVars(const StaticEnv& env) { abort(); } + +void ExprInt::bindVars(const StaticEnv& env) {} + +void ExprFloat::bindVars(const StaticEnv& env) {} + +void ExprString::bindVars(const StaticEnv& env) {} + +void ExprPath::bindVars(const StaticEnv& env) {} + +void ExprVar::bindVars(const StaticEnv& env) { + /* Check whether the variable appears in the environment. If so, + set its level and displacement. */ + const StaticEnv* curEnv; + unsigned int level; + std::optional<unsigned int> withLevel = std::nullopt; + for (curEnv = &env, level = 0; curEnv != nullptr; + curEnv = curEnv->up, level++) { + if (curEnv->isWith) { + if (!withLevel.has_value()) { + withLevel = level; + } + } else { + auto i = curEnv->vars.find(name); + if (i != curEnv->vars.end()) { + fromWith = false; + this->level = level; + displ = i->second; + return; + } + } + } + + /* Otherwise, the variable must be obtained from the nearest + enclosing `with'. If there is no `with', then we can issue an + "undefined variable" error now. */ + if (!withLevel.has_value()) { + throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % + pos); + } + + fromWith = true; + this->level = withLevel.value(); +} + +void ExprSelect::bindVars(const StaticEnv& env) { + e->bindVars(env); + if (def != nullptr) { + def->bindVars(env); + } + for (auto& i : attrPath) { + if (auto* expr = std::get_if<Expr*>(&i)) { + (*expr)->bindVars(env); + } + } +} + +void ExprOpHasAttr::bindVars(const StaticEnv& env) { + e->bindVars(env); + for (auto& i : attrPath) { + if (auto* expr = std::get_if<Expr*>(&i)) { + (*expr)->bindVars(env); + } + } +} + +void ExprAttrs::bindVars(const StaticEnv& env) { + const StaticEnv* dynamicEnv = &env; + StaticEnv newEnv(/* isWith = */ false, &env); + + if (recursive) { + dynamicEnv = &newEnv; + + unsigned int displ = 0; + for (auto& i : attrs) { + newEnv.vars[i.first] = i.second.displ = displ++; + } + + for (auto& i : attrs) { + i.second.e->bindVars(i.second.inherited ? env : newEnv); + } + } + + else { + for (auto& i : attrs) { + i.second.e->bindVars(env); + } + } + + for (auto& i : dynamicAttrs) { + i.nameExpr->bindVars(*dynamicEnv); + i.valueExpr->bindVars(*dynamicEnv); + } +} + +void ExprList::bindVars(const StaticEnv& env) { + for (auto& i : elems) { + i->bindVars(env); + } +} + +void ExprLambda::bindVars(const StaticEnv& env) { + StaticEnv newEnv(false, &env); + + unsigned int displ = 0; + + if (!arg.empty()) { + newEnv.vars[arg] = displ++; + } + + if (matchAttrs) { + for (auto& i : formals->formals) { + newEnv.vars[i.name] = displ++; + } + + for (auto& i : formals->formals) { + if (i.def != nullptr) { + i.def->bindVars(newEnv); + } + } + } + + body->bindVars(newEnv); +} + +void ExprLet::bindVars(const StaticEnv& env) { + StaticEnv newEnv(false, &env); + + unsigned int displ = 0; + for (auto& i : attrs->attrs) { + newEnv.vars[i.first] = i.second.displ = displ++; + } + + for (auto& i : attrs->attrs) { + i.second.e->bindVars(i.second.inherited ? env : newEnv); + } + + body->bindVars(newEnv); +} + +void ExprWith::bindVars(const StaticEnv& env) { + /* Does this `with' have an enclosing `with'? If so, record its + level so that `lookupVar' can look up variables in the previous + `with' if this one doesn't contain the desired attribute. */ + const StaticEnv* curEnv; + unsigned int level; + prevWith = 0; + for (curEnv = &env, level = 1; curEnv != nullptr; + curEnv = curEnv->up, level++) { + if (curEnv->isWith) { + prevWith = level; + break; + } + } + + attrs->bindVars(env); + StaticEnv newEnv(true, &env); + body->bindVars(newEnv); +} + +void ExprIf::bindVars(const StaticEnv& env) { + cond->bindVars(env); + then->bindVars(env); + else_->bindVars(env); +} + +void ExprAssert::bindVars(const StaticEnv& env) { + cond->bindVars(env); + body->bindVars(env); +} + +void ExprOpNot::bindVars(const StaticEnv& env) { e->bindVars(env); } + +void ExprConcatStrings::bindVars(const StaticEnv& env) { + for (auto& i : *es) { + i->bindVars(env); + } +} + +void ExprPos::bindVars(const StaticEnv& env) {} + +/* Storing function names. */ +void ExprLambda::setName(Symbol& name) { this->name = name; } + +std::string ExprLambda::showNamePos() const { + return (format("%1% at %2%") % + (name.has_value() ? "'" + std::string(name.value()) + "'" + : "anonymous function") % + pos) + .str(); +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/nixexpr.hh b/third_party/nix/src/libexpr/nixexpr.hh new file mode 100644 index 0000000000..16b58dec2e --- /dev/null +++ b/third_party/nix/src/libexpr/nixexpr.hh @@ -0,0 +1,361 @@ +#pragma once + +#include <map> +#include <optional> +#include <variant> + +#include <absl/container/flat_hash_map.h> + +#include "libexpr/symbol-table.hh" +#include "libexpr/value.hh" +#include "libutil/types.hh" // TODO(tazjin): audit this include + +namespace nix { + +MakeError(EvalError, Error); +MakeError(ParseError, Error); +MakeError(AssertionError, EvalError); +MakeError(ThrownError, AssertionError); +MakeError(Abort, EvalError); +MakeError(TypeError, EvalError); +MakeError(UndefinedVarError, Error); +MakeError(RestrictedPathError, Error); + +/* Position objects. */ + +struct Pos { + std::optional<Symbol> file; + unsigned int line, column; + Pos(const std::optional<Symbol>& file, unsigned int line, unsigned int column) + : file(file), line(line), column(column){}; + + // TODO(tazjin): remove this - empty pos is never useful + Pos() : file(std::nullopt), line(0), column(0){}; + + operator bool() const { return line != 0; } + + bool operator<(const Pos& p2) const { + if (!file.has_value()) { + return true; + } + + if (!line) { + return p2.line; + } + if (!p2.line) { + return false; + } + int d = ((std::string)file.value()).compare((std::string)p2.file.value()); + if (d < 0) { + return true; + } + if (d > 0) { + return false; + } + if (line < p2.line) { + return true; + } + if (line > p2.line) { + return false; + } + return column < p2.column; + } +}; + +extern Pos noPos; + +std::ostream& operator<<(std::ostream& str, const Pos& pos); + +struct Env; +struct Value; +class EvalState; +struct StaticEnv; + +/* An attribute path is a sequence of attribute names. */ +using AttrName = std::variant<Symbol, Expr*>; +using AttrPath = std::vector<AttrName>; +using AttrNameVector = std::vector<AttrName>; + +using VectorExprs = std::vector<nix::Expr*>; + +std::string showAttrPath(const AttrPath& attrPath); + +/* Abstract syntax of Nix expressions. */ + +struct Expr { + virtual ~Expr(){}; + virtual void show(std::ostream& str) const; + virtual void bindVars(const StaticEnv& env); + virtual void eval(EvalState& state, Env& env, Value& v); + virtual Value* maybeThunk(EvalState& state, Env& env); +}; + +std::ostream& operator<<(std::ostream& str, const Expr& e); + +#define COMMON_METHODS \ + void show(std::ostream& str) const; \ + void eval(EvalState& state, Env& env, Value& v); \ + void bindVars(const StaticEnv& env); + +struct ExprInt : Expr { + NixInt n; + Value v; + ExprInt(NixInt n) : n(n) { mkInt(v, n); }; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); +}; + +struct ExprFloat : Expr { + NixFloat nf; + Value v; + ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); }; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); +}; + +struct ExprString : Expr { + Symbol s; + Value v; + ExprString(const Symbol& s) : s(s) { mkString(v, s); }; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); +}; + +/* Temporary class used during parsing of indented strings. */ +struct ExprIndStr : Expr { + std::string s; + ExprIndStr(const std::string& s) : s(s){}; +}; + +struct ExprPath : Expr { + std::string s; + Value v; + ExprPath(const std::string& s) : s(s) { mkPathNoCopy(v, this->s.c_str()); }; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); +}; + +struct ExprVar : Expr { + Pos pos; + Symbol name; + + /* Whether the variable comes from an environment (e.g. a rec, let + or function argument) or from a "with". */ + bool fromWith; + + /* In the former case, the value is obtained by going `level' + levels up from the current environment and getting the + `displ'th value in that environment. In the latter case, the + value is obtained by getting the attribute named `name' from + the set stored in the environment that is `level' levels up + from the current one.*/ + unsigned int level; + unsigned int displ; + + ExprVar(const Symbol& name) : name(name){}; + ExprVar(const Pos& pos, const Symbol& name) : pos(pos), name(name){}; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); +}; + +// [tazjin] I *think* that this struct describes the syntactic +// construct for "selecting" something out of an attribute set, e.g. +// `a.b.c` => ExprSelect{"b", "c"}. +// +// Each path element has got a pointer to an expression, which seems +// to be the thing preceding its period, but afaict that is only set +// for the first one in a path. +struct ExprSelect : Expr { + Pos pos; + Expr *e, *def; + AttrPath attrPath; + ExprSelect(const Pos& pos, Expr* e, const AttrPath& attrPath, Expr* def) + : pos(pos), e(e), def(def), attrPath(attrPath){}; + ExprSelect(const Pos& pos, Expr* e, const Symbol& name) + : pos(pos), e(e), def(0) { + attrPath.push_back(AttrName(name)); + }; + COMMON_METHODS +}; + +struct ExprOpHasAttr : Expr { + Pos pos; + Expr* e; + AttrPath attrPath; + ExprOpHasAttr(Expr* e, const AttrPath& attrPath) : e(e), attrPath(attrPath){}; + ExprOpHasAttr(const Pos& pos, Expr* e, const AttrPath& attrPath) + : pos(pos), e(e), attrPath(attrPath){}; + COMMON_METHODS +}; + +struct ExprAttrs : Expr { + bool recursive; + + struct AttrDef { + bool inherited; + Expr* e; + Pos pos; + unsigned int displ; // displacement + AttrDef(Expr* e, const Pos& pos, bool inherited = false) + : inherited(inherited), e(e), pos(pos), displ(0){}; + AttrDef(){}; + }; + + using AttrDefs = absl::flat_hash_map<Symbol, AttrDef>; + AttrDefs attrs; + + struct DynamicAttrDef { + Expr *nameExpr, *valueExpr; + Pos pos; + DynamicAttrDef(Expr* nameExpr, Expr* valueExpr, const Pos& pos) + : nameExpr(nameExpr), valueExpr(valueExpr), pos(pos){}; + }; + + using DynamicAttrDefs = std::vector<DynamicAttrDef>; + DynamicAttrDefs dynamicAttrs; + + ExprAttrs() : recursive(false){}; + COMMON_METHODS +}; + +struct ExprList : Expr { + VectorExprs elems; + ExprList(){}; + COMMON_METHODS +}; + +struct Formal { + Symbol name; + Expr* def; // def = default, not definition + Formal(const Symbol& name, Expr* def) : name(name), def(def){}; +}; + +// Describes structured function arguments (e.g. `{ a }: ...`) +struct Formals { + using Formals_ = std::list<Formal>; + Formals_ formals; + std::set<Symbol> argNames; // used during parsing + bool ellipsis; +}; + +struct ExprLambda : Expr { + public: + Pos pos; + std::optional<Symbol> name; + Symbol arg; + bool matchAttrs; + Formals* formals; + Expr* body; + ExprLambda(const Pos& pos, const Symbol& arg, bool matchAttrs, + Formals* formals, Expr* body) + : pos(pos), + arg(arg), + matchAttrs(matchAttrs), + formals(formals), + body(body) { + if (!arg.empty() && formals && + formals->argNames.find(arg) != formals->argNames.end()) { + throw ParseError( + format("duplicate formal function argument '%1%' at %2%") % arg % + pos); + } + }; + void setName(Symbol& name); + std::string showNamePos() const; + COMMON_METHODS +}; + +struct ExprLet : Expr { + ExprAttrs* attrs; + Expr* body; + ExprLet(ExprAttrs* attrs, Expr* body) : attrs(attrs), body(body){}; + COMMON_METHODS +}; + +struct ExprWith : Expr { + Pos pos; + Expr *attrs, *body; + size_t prevWith; + ExprWith(const Pos& pos, Expr* attrs, Expr* body) + : pos(pos), attrs(attrs), body(body){}; + COMMON_METHODS +}; + +struct ExprIf : Expr { + Pos pos; + Expr *cond, *then, *else_; + ExprIf(Expr* cond, Expr* then, Expr* else_) + : cond(cond), then(then), else_(else_){}; + ExprIf(const Pos& pos, Expr* cond, Expr* then, Expr* else_) + : pos(pos), cond(cond), then(then), else_(else_){}; + COMMON_METHODS +}; + +struct ExprAssert : Expr { + Pos pos; + Expr *cond, *body; + ExprAssert(const Pos& pos, Expr* cond, Expr* body) + : pos(pos), cond(cond), body(body){}; + COMMON_METHODS +}; + +struct ExprOpNot : Expr { + Pos pos; + Expr* e; + explicit ExprOpNot(Expr* e) : e(e){}; + ExprOpNot(const Pos& pos, Expr* e) : pos(pos), e(e){}; + COMMON_METHODS +}; + +#define MakeBinOp(name, s) \ + struct name : Expr { \ + Pos pos; \ + Expr *e1, *e2; \ + name(Expr* e1, Expr* e2) : e1(e1), e2(e2){}; \ + name(const Pos& pos, Expr* e1, Expr* e2) : pos(pos), e1(e1), e2(e2){}; \ + void show(std::ostream& str) const { \ + str << "(" << *e1 << " " s " " << *e2 << ")"; \ + } \ + void bindVars(const StaticEnv& env) { \ + e1->bindVars(env); \ + e2->bindVars(env); \ + } \ + void eval(EvalState& state, Env& env, Value& v); \ + }; + +MakeBinOp(ExprApp, ""); +MakeBinOp(ExprOpEq, "=="); +MakeBinOp(ExprOpNEq, "!="); +MakeBinOp(ExprOpAnd, "&&"); +MakeBinOp(ExprOpOr, "||"); +MakeBinOp(ExprOpImpl, "->"); +MakeBinOp(ExprOpUpdate, "//"); +MakeBinOp(ExprOpConcatLists, "++"); + +struct ExprConcatStrings : Expr { + Pos pos; + bool forceString; + nix::VectorExprs* es; + ExprConcatStrings(const Pos& pos, bool forceString, nix::VectorExprs* es) + : pos(pos), forceString(forceString), es(es){}; + COMMON_METHODS +}; + +struct ExprPos : Expr { + Pos pos; + ExprPos(const Pos& pos) : pos(pos){}; + COMMON_METHODS +}; + +/* Static environments are used to map variable names onto (level, + displacement) pairs used to obtain the value of the variable at + runtime. */ +struct StaticEnv { + bool isWith; + const StaticEnv* up; + typedef absl::flat_hash_map<Symbol, unsigned int> Vars; + Vars vars; + StaticEnv(bool isWith, const StaticEnv* up) : isWith(isWith), up(up){}; +}; + +} // namespace nix diff --git a/third_party/nix/src/libexpr/parser.cc b/third_party/nix/src/libexpr/parser.cc new file mode 100644 index 0000000000..aea6cec7e4 --- /dev/null +++ b/third_party/nix/src/libexpr/parser.cc @@ -0,0 +1,332 @@ +#include "libexpr/parser.hh" + +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "libexpr/eval.hh" +#include "libstore/download.hh" +#include "libstore/store-api.hh" + +namespace nix { + +void addAttr(ExprAttrs* attrs, AttrPath& attrPath, Expr* e, const Pos& pos) { + AttrPath::iterator i; + // All attrpaths have at least one attr + assert(!attrPath.empty()); + // Checking attrPath validity. + // =========================== + for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + if (const auto* sym = std::get_if<Symbol>(&(*i)); sym && sym->set()) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(*sym); + if (j != attrs->attrs.end()) { + if (!j->second.inherited) { + ExprAttrs* attrs2 = dynamic_cast<ExprAttrs*>(j->second.e); + if (!attrs2) { + dupAttr(attrPath, pos, j->second.pos); + } + attrs = attrs2; + } else { + dupAttr(attrPath, pos, j->second.pos); + } + } else { + ExprAttrs* nested = new ExprAttrs; + attrs->attrs[*sym] = ExprAttrs::AttrDef(nested, pos); + attrs = nested; + } + } else { + // Yes, this code does not handle all conditions + // exhaustively. We use std::get to throw if the condition + // that isn't covered happens, which is potentially a + // behaviour change from the previous default constructed + // Symbol. It should alert us about anything untoward going + // on here. + auto* expr = std::get<Expr*>(*i); + + ExprAttrs* nested = new ExprAttrs; + attrs->dynamicAttrs.push_back( + ExprAttrs::DynamicAttrDef(expr, nested, pos)); + attrs = nested; + } + } + // Expr insertion. + // ========================== + if (auto* sym = std::get_if<Symbol>(&(*i)); sym && sym->set()) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(*sym); + if (j != attrs->attrs.end()) { + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + auto ae = dynamic_cast<ExprAttrs*>(e); + auto jAttrs = dynamic_cast<ExprAttrs*>(j->second.e); + if (jAttrs && ae) { + for (auto& ad : ae->attrs) { + auto j2 = jAttrs->attrs.find(ad.first); + if (j2 != + jAttrs->attrs.end()) { // Attr already defined in iAttrs, error. + dupAttr(ad.first, j2->second.pos, ad.second.pos); + } + jAttrs->attrs[ad.first] = ad.second; + } + } else { + dupAttr(attrPath, pos, j->second.pos); + } + } else { + // This attr path is not defined. Let's create it. + attrs->attrs[*sym] = ExprAttrs::AttrDef(e, pos); + } + } else { + // Same caveat as the identical line above. + auto* expr = std::get<Expr*>(*i); + attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(expr, e, pos)); + } +} + +void addFormal(const Pos& pos, Formals* formals, const Formal& formal) { + if (formals->argNames.find(formal.name) != formals->argNames.end()) { + throw ParseError(format("duplicate formal function argument '%1%' at %2%") % + formal.name % pos); + } + formals->formals.push_front(formal); + formals->argNames.insert(formal.name); +} + +Expr* stripIndentation(const Pos& pos, SymbolTable& symbols, VectorExprs& es) { + if (es.empty()) { + return new ExprString(symbols.Create("")); + } + + /* Figure out the minimum indentation. Note that by design + whitespace-only final lines are not taken into account. (So + the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ + bool atStartOfLine = true; /* = seen only whitespace in the current line */ + size_t minIndent = 1000000; + size_t curIndent = 0; + for (auto& i : es) { + ExprIndStr* e = dynamic_cast<ExprIndStr*>(i); + if (!e) { + /* Anti-quotations end the current start-of-line whitespace. */ + if (atStartOfLine) { + atStartOfLine = false; + if (curIndent < minIndent) { + minIndent = curIndent; + } + } + continue; + } + for (size_t j = 0; j < e->s.size(); ++j) { + if (atStartOfLine) { + if (e->s[j] == ' ') { + curIndent++; + } else if (e->s[j] == '\n') { + /* Empty line, doesn't influence minimum + indentation. */ + curIndent = 0; + } else { + atStartOfLine = false; + if (curIndent < minIndent) { + minIndent = curIndent; + } + } + } else if (e->s[j] == '\n') { + atStartOfLine = true; + curIndent = 0; + } + } + } + + /* Strip spaces from each line. */ + VectorExprs* es2 = new VectorExprs; + atStartOfLine = true; + size_t curDropped = 0; + size_t n = es.size(); + for (VectorExprs::iterator i = es.begin(); i != es.end(); ++i, --n) { + ExprIndStr* e = dynamic_cast<ExprIndStr*>(*i); + if (!e) { + atStartOfLine = false; + curDropped = 0; + es2->push_back(*i); + continue; + } + + std::string s2; + for (size_t j = 0; j < e->s.size(); ++j) { + if (atStartOfLine) { + if (e->s[j] == ' ') { + if (curDropped++ >= minIndent) { + s2 += e->s[j]; + } + } else if (e->s[j] == '\n') { + curDropped = 0; + s2 += e->s[j]; + } else { + atStartOfLine = false; + curDropped = 0; + s2 += e->s[j]; + } + } else { + s2 += e->s[j]; + if (e->s[j] == '\n') { + atStartOfLine = true; + } + } + } + + /* Remove the last line if it is empty and consists only of + spaces. */ + if (n == 1) { + std::string::size_type p = s2.find_last_of('\n'); + if (p != std::string::npos && + s2.find_first_not_of(' ', p + 1) == std::string::npos) { + s2 = std::string(s2, 0, p + 1); + } + } + + es2->push_back(new ExprString(symbols.Create(s2))); + } + + /* If this is a single string, then don't do a concatenation. */ + return es2->size() == 1 && dynamic_cast<ExprString*>((*es2)[0]) + ? (*es2)[0] + : new ExprConcatStrings(pos, true, es2); +} + +Path resolveExprPath(Path path) { + assert(path[0] == '/'); + + /* If `path' is a symlink, follow it. This is so that relative + path references work. */ + struct stat st; + while (true) { + if (lstat(path.c_str(), &st)) { + throw SysError(format("getting status of '%1%'") % path); + } + if (!S_ISLNK(st.st_mode)) { + break; + } + path = absPath(readLink(path), dirOf(path)); + } + + /* If `path' refers to a directory, append `/default.nix'. */ + if (S_ISDIR(st.st_mode)) { + path = canonPath(path + "/default.nix"); + } + + return path; +} + +// These methods are actually declared in eval.hh, and were - for some +// reason - previously implemented in parser.y. + +Expr* EvalState::parseExprFromFile(const Path& path) { + return parseExprFromFile(path, staticBaseEnv); +} + +Expr* EvalState::parseExprFromFile(const Path& path, StaticEnv& staticEnv) { + return parse(readFile(path).c_str(), path, dirOf(path), staticEnv); +} + +Expr* EvalState::parseExprFromString(const std::string& s, const Path& basePath, + StaticEnv& staticEnv) { + return parse(s.c_str(), "(std::string)", basePath, staticEnv); +} + +Expr* EvalState::parseExprFromString(const std::string& s, + const Path& basePath) { + return parseExprFromString(s, basePath, staticBaseEnv); +} + +Expr* EvalState::parseStdin() { + // Activity act(*logger, lvlTalkative, format("parsing standard input")); + return parseExprFromString(drainFD(0), absPath(".")); +} + +void EvalState::addToSearchPath(const std::string& s) { + size_t pos = s.find('='); + std::string prefix; + Path path; + if (pos == std::string::npos) { + path = s; + } else { + prefix = std::string(s, 0, pos); + path = std::string(s, pos + 1); + } + + searchPath.emplace_back(prefix, path); +} + +Path EvalState::findFile(const std::string& path) { + return findFile(searchPath, path); +} + +Path EvalState::findFile(SearchPath& searchPath, const std::string& path, + const Pos& pos) { + for (auto& i : searchPath) { + std::string suffix; + if (i.first.empty()) { + suffix = "/" + path; + } else { + auto s = i.first.size(); + if (path.compare(0, s, i.first) != 0 || + (path.size() > s && path[s] != '/')) { + continue; + } + suffix = path.size() == s ? "" : "/" + std::string(path, s); + } + auto r = resolveSearchPathElem(i); + if (!r.first) { + continue; + } + Path res = r.second + suffix; + if (pathExists(res)) { + return canonPath(res); + } + } + format f = format( + "file '%1%' was not found in the Nix search path (add it using $NIX_PATH " + "or -I)" + + std::string(pos ? ", at %2%" : "")); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + throw ThrownError(f % path % pos); +} + +std::pair<bool, std::string> EvalState::resolveSearchPathElem( + const SearchPathElem& elem) { + auto i = searchPathResolved.find(elem.second); + if (i != searchPathResolved.end()) { + return i->second; + } + + std::pair<bool, std::string> res; + + if (isUri(elem.second)) { + try { + CachedDownloadRequest request(elem.second); + request.unpack = true; + res = {true, getDownloader()->downloadCached(store, request).path}; + } catch (DownloadError& e) { + LOG(WARNING) << "Nix search path entry '" << elem.second + << "' cannot be downloaded, ignoring"; + res = {false, ""}; + } + } else { + auto path = absPath(elem.second); + if (pathExists(path)) { + res = {true, path}; + } else { + LOG(WARNING) << "Nix search path entry '" << elem.second + << "' does not exist, ignoring"; + res = {false, ""}; + } + } + + VLOG(2) << "resolved search path element '" << elem.second << "' to '" + << res.second << "'"; + + searchPathResolved[elem.second] = res; + return res; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/parser.hh b/third_party/nix/src/libexpr/parser.hh new file mode 100644 index 0000000000..70b5450b5a --- /dev/null +++ b/third_party/nix/src/libexpr/parser.hh @@ -0,0 +1,100 @@ +// Parser utilities for use in parser.y +#pragma once + +// TODO(tazjin): Audit these includes, they were in parser.y +#include <optional> +#include <variant> + +#include <glog/logging.h> + +#include "libexpr/eval.hh" +#include "libexpr/nixexpr.hh" +#include "libutil/util.hh" + +#define YY_DECL \ + int yylex(YYSTYPE* yylval_param, YYLTYPE* yylloc_param, yyscan_t yyscanner, \ + nix::ParseData* data) + +#define CUR_POS makeCurPos(*yylocp, data) + +namespace nix { + +struct ParseData { + EvalState& state; + SymbolTable& symbols; + Expr* result; + Path basePath; + std::optional<Symbol> path; + std::string error; + Symbol sLetBody; + + ParseData(EvalState& state) + : state(state), + symbols(state.symbols), + sLetBody(symbols.Create("<let-body>")){}; +}; + +// Clang fails to identify these functions as used, probably because +// of some interaction between the lexer/parser codegen and something +// else. +// +// To avoid warnings for that we disable -Wunused-function in this block. + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-function" + +// TODO(tazjin): move dupAttr to anonymous namespace +static void dupAttr(const AttrPath& attrPath, const Pos& pos, + const Pos& prevPos) { + throw ParseError(format("attribute '%1%' at %2% already defined at %3%") % + showAttrPath(attrPath) % pos % prevPos); +} + +static void dupAttr(Symbol attr, const Pos& pos, const Pos& prevPos) { + throw ParseError(format("attribute '%1%' at %2% already defined at %3%") % + attr % pos % prevPos); +} + +void addAttr(ExprAttrs* attrs, AttrPath& attrPath, Expr* e, const Pos& pos); + +void addFormal(const Pos& pos, Formals* formals, const Formal& formal); + +Expr* stripIndentation(const Pos& pos, SymbolTable& symbols, VectorExprs& es); + +Path resolveExprPath(Path path); + +// implementations originally from lexer.l + +static Expr* unescapeStr(SymbolTable& symbols, const char* s, size_t length) { + std::string t; + t.reserve(length); + char c; + while ((c = *s++)) { + if (c == '\\') { + assert(*s); + c = *s++; + if (c == 'n') { + t += '\n'; + } else if (c == 'r') { + t += '\r'; + } else if (c == 't') { + t += '\t'; + } else { + t += c; + } + } else if (c == '\r') { + /* Normalise CR and CR/LF into LF. */ + t += '\n'; + if (*s == '\n') { + s++; + } /* cr/lf */ + } else { + t += c; + } + } + return new ExprString(symbols.Create(t)); +} + +#pragma clang diagnostic pop // re-enable -Wunused-function + +} // namespace nix diff --git a/third_party/nix/src/libexpr/parser.y b/third_party/nix/src/libexpr/parser.y new file mode 100644 index 0000000000..a8af06802f --- /dev/null +++ b/third_party/nix/src/libexpr/parser.y @@ -0,0 +1,359 @@ +%glr-parser +%locations +%define parse.error verbose +%define api.pure true +%defines +/* %no-lines */ +%parse-param { void * scanner } +%parse-param { nix::ParseData * data } +%lex-param { void * scanner } +%lex-param { nix::ParseData * data } +%expect 1 +%expect-rr 1 + +%code requires { +#define YY_NO_INPUT 1 // disable unused yyinput features +#include "libexpr/parser.hh" + +struct YYSTYPE { + union { + nix::Expr * e; + nix::ExprList * list; + nix::ExprAttrs * attrs; + nix::Formals * formals; + nix::Formal * formal; + nix::NixInt n; + nix::NixFloat nf; + const char * id; // !!! -> Symbol + char * path; + char * uri; + nix::AttrNameVector * attrNames; + nix::VectorExprs * string_parts; + }; +}; + +} + +%{ + +#include "generated/parser-tab.hh" +#include "generated/lexer-tab.hh" + +YY_DECL; + +using namespace nix; + +namespace nix { + +static inline Pos makeCurPos(const YYLTYPE& loc, ParseData* data) { + return Pos(data->path, loc.first_line, loc.first_column); +} + +void yyerror(YYLTYPE* loc, yyscan_t scanner, ParseData* data, + const char* error) { + data->error = (format("%1%, at %2%") % error % makeCurPos(*loc, data)).str(); +} + +} + +%} + +%type <e> start expr expr_function expr_if expr_op +%type <e> expr_app expr_select expr_simple +%type <list> expr_list +%type <attrs> binds +%type <formals> formals +%type <formal> formal +%type <attrNames> attrs attrpath +%type <string_parts> string_parts_interpolated ind_string_parts +%type <e> string_parts string_attr +%type <id> attr +%token <id> ID ATTRPATH +%token <e> STR IND_STR +%token <n> INT +%token <nf> FLOAT +%token <path> PATH HPATH SPATH +%token <uri> URI +%token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW +%token DOLLAR_CURLY /* == ${ */ +%token IND_STRING_OPEN IND_STRING_CLOSE +%token ELLIPSIS + +%right IMPL +%left OR +%left AND +%nonassoc EQ NEQ +%nonassoc '<' '>' LEQ GEQ +%right UPDATE +%left NOT +%left '+' '-' +%left '*' '/' +%right CONCAT +%nonassoc '?' +%nonassoc NEGATE + +%% + +start: expr { data->result = $1; }; + +expr: expr_function; + +expr_function + : ID ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.Create($1), false, 0, $3); } + | '{' formals '}' ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.Create(""), true, $2, $5); } + | '{' formals '}' '@' ID ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.Create($5), true, $2, $7); } + | ID '@' '{' formals '}' ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.Create($1), true, $4, $7); } + | ASSERT expr ';' expr_function + { $$ = new ExprAssert(CUR_POS, $2, $4); } + | WITH expr ';' expr_function + { $$ = new ExprWith(CUR_POS, $2, $4); } + | LET binds IN expr_function + { if (!$2->dynamicAttrs.empty()) + throw ParseError(format("dynamic attributes not allowed in let at %1%") + % CUR_POS); + $$ = new ExprLet($2, $4); + } + | expr_if + ; + +expr_if + : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); } + | expr_op + ; + +expr_op + : '!' expr_op %prec NOT { $$ = new ExprOpNot(CUR_POS, $2); } + | '-' expr_op %prec NEGATE { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__sub")), new ExprInt(0)), $2); } + | expr_op EQ expr_op { $$ = new ExprOpEq(CUR_POS, $1, $3); } + | expr_op NEQ expr_op { $$ = new ExprOpNEq(CUR_POS, $1, $3); } + | expr_op '<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__lessThan")), $1), $3); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__lessThan")), $3), $1)); } + | expr_op '>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__lessThan")), $3), $1); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__lessThan")), $1), $3)); } + | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); } + | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); } + | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); } + | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(CUR_POS, $1, $3); } + | expr_op '?' attrpath { $$ = new ExprOpHasAttr(CUR_POS, $1, *$3); } + | expr_op '+' expr_op + { $$ = new ExprConcatStrings(CUR_POS, false, new nix::VectorExprs({$1, $3})); } + | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__sub")), $1), $3); } + | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__mul")), $1), $3); } + | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__div")), $1), $3); } + | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); } + | expr_app + ; + +expr_app + : expr_app expr_select + { $$ = new ExprApp(CUR_POS, $1, $2); } + | expr_select { $$ = $1; } + ; + +expr_select + : expr_simple '.' attrpath + { $$ = new ExprSelect(CUR_POS, $1, *$3, 0); } + | expr_simple '.' attrpath OR_KW expr_select + { $$ = new ExprSelect(CUR_POS, $1, *$3, $5); } + | /* Backwards compatibility: because Nixpkgs has a rarely used + function named ‘or’, allow stuff like ‘map or [...]’. */ + expr_simple OR_KW + { $$ = new ExprApp(CUR_POS, $1, new ExprVar(CUR_POS, data->symbols.Create("or"))); } + | expr_simple { $$ = $1; } + ; + +expr_simple + : ID { + if (strcmp($1, "__curPos") == 0) + $$ = new ExprPos(CUR_POS); + else + $$ = new ExprVar(CUR_POS, data->symbols.Create($1)); + } + | INT { $$ = new ExprInt($1); } + | FLOAT { $$ = new ExprFloat($1); } + | '"' string_parts '"' { $$ = $2; } + | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { + $$ = stripIndentation(CUR_POS, data->symbols, *$2); + } + | PATH { $$ = new ExprPath(absPath($1, data->basePath)); } + | HPATH { $$ = new ExprPath(getHome() + std::string{$1 + 1}); } + | SPATH { + std::string path($1 + 1, strlen($1) - 2); + $$ = new ExprApp(CUR_POS, + new ExprApp(new ExprVar(data->symbols.Create("__findFile")), + new ExprVar(data->symbols.Create("__nixPath"))), + new ExprString(data->symbols.Create(path))); + } + | URI { $$ = new ExprString(data->symbols.Create($1)); } + | '(' expr ')' { $$ = $2; } + /* Let expressions `let {..., body = ...}' are just desugared + into `(rec {..., body = ...}).body'. */ + | LET '{' binds '}' + { $3->recursive = true; $$ = new ExprSelect(noPos, $3, data->symbols.Create("body")); } + | REC '{' binds '}' + { $3->recursive = true; $$ = $3; } + | '{' binds '}' + { $$ = $2; } + | '[' expr_list ']' { $$ = $2; } + ; + +string_parts + : STR + | string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, $1); } + | { $$ = new ExprString(data->symbols.Create("")); } + ; + +string_parts_interpolated + : string_parts_interpolated STR { $$ = $1; $1->push_back($2); } + | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); } + | DOLLAR_CURLY expr '}' { $$ = new nix::VectorExprs; $$->push_back($2); } + | STR DOLLAR_CURLY expr '}' { + $$ = new nix::VectorExprs; + $$->push_back($1); + $$->push_back($3); + } + ; + +ind_string_parts + : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); } + | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); } + | { $$ = new nix::VectorExprs; } + ; + +binds + : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, *$2, $4, makeCurPos(@2, data)); } + | binds INHERIT attrs ';' + { $$ = $1; + for (auto & i : *$3) { + auto sym = std::get<Symbol>(i); + if ($$->attrs.find(sym) != $$->attrs.end()) { + dupAttr(sym, makeCurPos(@3, data), $$->attrs[sym].pos); + } + Pos pos = makeCurPos(@3, data); + $$->attrs[sym] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, sym), pos, true); + } + } + | binds INHERIT '(' expr ')' attrs ';' + { $$ = $1; + /* !!! Should ensure sharing of the expression in $4. */ + for (auto & i : *$6) { + auto sym = std::get<Symbol>(i); + if ($$->attrs.find(sym) != $$->attrs.end()) { + dupAttr(sym, makeCurPos(@6, data), $$->attrs[sym].pos); + } + $$->attrs[sym] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, sym), makeCurPos(@6, data)); + } + } + | { $$ = new ExprAttrs; } + ; + +attrs + : attrs attr { $$ = $1; $1->push_back(AttrName(data->symbols.Create($2))); } + | attrs string_attr + { $$ = $1; + ExprString * str = dynamic_cast<ExprString *>($2); + if (str) { + $$->push_back(AttrName(str->s)); + delete str; + } else + throw ParseError(format("dynamic attributes not allowed in inherit at %1%") + % makeCurPos(@2, data)); + } + | { $$ = new AttrPath; } + ; + +attrpath + : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(data->symbols.Create($3))); } + | attrpath '.' string_attr + { $$ = $1; + ExprString * str = dynamic_cast<ExprString *>($3); + if (str) { + $$->push_back(AttrName(str->s)); + delete str; + } else { + $$->push_back(AttrName($3)); + } + } + | attr { $$ = new nix::AttrNameVector; $$->push_back(AttrName(data->symbols.Create($1))); } + | string_attr + { $$ = new nix::AttrNameVector; + ExprString *str = dynamic_cast<ExprString *>($1); + if (str) { + $$->push_back(AttrName(str->s)); + delete str; + } else + $$->push_back(AttrName($1)); + } + ; + +attr + : ID { $$ = $1; } + | OR_KW { $$ = "or"; } + ; + +string_attr + : '"' string_parts '"' { $$ = $2; } + | DOLLAR_CURLY expr '}' { $$ = $2; } + ; + +expr_list + : expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */ } + | { $$ = new ExprList; } + ; + +formals + : formal ',' formals + { $$ = $3; addFormal(CUR_POS, $$, *$1); } + | formal + { $$ = new Formals; addFormal(CUR_POS, $$, *$1); $$->ellipsis = false; } + | + { $$ = new Formals; $$->ellipsis = false; } + | ELLIPSIS + { $$ = new Formals; $$->ellipsis = true; } + ; + +formal + : ID { $$ = new Formal(data->symbols.Create($1), 0); } + | ID '?' expr { $$ = new Formal(data->symbols.Create($1), $3); } + ; + +%% + + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> + +#include "libexpr/eval.hh" +#include "libstore/store-api.hh" + + +namespace nix { + +Expr* EvalState::parse(const char* text, const Path& path, const Path& basePath, + StaticEnv& staticEnv) { + yyscan_t scanner; + ParseData data(*this); + data.basePath = basePath; + data.path = data.symbols.Create(path); + + yylex_init(&scanner); + yy_scan_string(text, scanner); + int res = yyparse(scanner, &data); + yylex_destroy(scanner); + + if (res) { + throw ParseError(data.error); + } + + data.result->bindVars(staticEnv); + + return data.result; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops.cc b/third_party/nix/src/libexpr/primops.cc new file mode 100644 index 0000000000..f196c5ed72 --- /dev/null +++ b/third_party/nix/src/libexpr/primops.cc @@ -0,0 +1,2335 @@ +#include "libexpr/primops.hh" + +#include <algorithm> +#include <cstring> +#include <iostream> +#include <regex> + +#include <absl/strings/str_split.h> +#include <glog/logging.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "libexpr/eval-inline.hh" +#include "libexpr/eval.hh" +#include "libexpr/json-to-value.hh" +#include "libexpr/names.hh" +#include "libexpr/value-to-json.hh" +#include "libexpr/value-to-xml.hh" +#include "libstore/derivations.hh" +#include "libstore/download.hh" +#include "libstore/globals.hh" +#include "libstore/store-api.hh" +#include "libutil/archive.hh" +#include "libutil/json.hh" +#include "libutil/status.hh" +#include "libutil/util.hh" + +namespace nix { + +/************************************************************* + * Miscellaneous + *************************************************************/ + +/* Decode a context string ‘!<name>!<path>’ into a pair <path, + name>. */ +std::pair<std::string, std::string> decodeContext(const std::string& s) { + if (s.at(0) == '!') { + size_t index = s.find('!', 1); + return std::pair<std::string, std::string>(std::string(s, index + 1), + std::string(s, 1, index - 1)); + } + return std::pair<std::string, std::string>( + s.at(0) == '/' ? s : std::string(s, 1), ""); +} + +InvalidPathError::InvalidPathError(const Path& path) + : EvalError(format("path '%1%' is not valid") % path), path(path) {} + +void EvalState::realiseContext(const PathSet& context) { + PathSet drvs; + + for (auto& i : context) { + std::pair<std::string, std::string> decoded = decodeContext(i); + Path ctx = decoded.first; + assert(store->isStorePath(ctx)); + if (!store->isValidPath(ctx)) { + throw InvalidPathError(ctx); + } + if (!decoded.second.empty() && nix::isDerivation(ctx)) { + drvs.insert(decoded.first + "!" + decoded.second); + + /* Add the output of this derivation to the allowed + paths. */ + if (allowedPaths) { + auto drv = store->derivationFromPath(decoded.first); + auto i = drv.outputs.find(decoded.second); + if (i == drv.outputs.end()) { + throw Error("derivation '%s' does not have an output named '%s'", + decoded.first, decoded.second); + } + allowedPaths->insert(i->second.path); + } + } + } + + if (drvs.empty()) { + return; + } + + if (!evalSettings.enableImportFromDerivation) { + throw EvalError(format("attempted to realize '%1%' during evaluation but " + "'allow-import-from-derivation' is false") % + *(drvs.begin())); + } + + /* For performance, prefetch all substitute info. */ + PathSet willBuild; + PathSet willSubstitute; + PathSet unknown; + unsigned long long downloadSize; + unsigned long long narSize; + store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, + narSize); + + nix::util::OkOrThrow(store->buildPaths(std::cerr, drvs)); +} + +/* Load and evaluate an expression from path specified by the + argument. */ +static void prim_scopedImport(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + + try { + state.realiseContext(context); + } catch (InvalidPathError& e) { + throw EvalError( + format("cannot import '%1%', since path '%2%' is not valid, at %3%") % + path % e.path % pos); + } + + Path realPath = state.checkSourcePath(state.toRealPath(path, context)); + + if (state.store->isStorePath(path) && state.store->isValidPath(path) && + isDerivation(path)) { + Derivation drv = readDerivation(realPath); + Value& w = *state.allocValue(); + state.mkAttrs(w, 3 + drv.outputs.size()); + Value* v2 = state.allocAttr(w, state.sDrvPath); + mkString(*v2, path, {"=" + path}); + v2 = state.allocAttr(w, state.sName); + mkString(*v2, drv.env["name"]); + Value* outputsVal = state.allocAttr(w, state.symbols.Create("outputs")); + state.mkList(*outputsVal, drv.outputs.size()); + unsigned int outputs_index = 0; + + for (const auto& o : drv.outputs) { + v2 = state.allocAttr(w, state.symbols.Create(o.first)); + mkString(*v2, o.second.path, {"!" + o.first + "!" + path}); + (*outputsVal->list)[outputs_index] = state.allocValue(); + mkString(*((*outputsVal->list)[outputs_index++]), o.first); + } + + Value fun; + state.evalFile( + settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", + fun); + state.forceFunction(fun, pos); + mkApp(v, fun, w); + state.forceAttrs(v, pos); + } else { + state.forceAttrs(*args[0]); + if (args[0]->attrs->empty()) { + state.evalFile(realPath, v); + } else { + Env* env = &state.allocEnv(args[0]->attrs->size()); + env->up = &state.baseEnv; + + StaticEnv staticEnv(false, &state.staticBaseEnv); + + unsigned int displ = 0; + for (auto& attr : *args[0]->attrs) { + staticEnv.vars[attr.second.name] = displ; + env->values[displ++] = attr.second.value; + } + + DLOG(INFO) << "evaluating file '" << realPath << "'"; + Expr* e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv); + + e->eval(state, *env, v); + } + } +} + +/* Return a string representing the type of the expression. */ +static void prim_typeOf(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + std::string t; + switch (args[0]->type) { + case tInt: + t = "int"; + break; + case tBool: + t = "bool"; + break; + case tString: + t = "string"; + break; + case tPath: + t = "path"; + break; + case tNull: + t = "null"; + break; + case tAttrs: + t = "set"; + break; + case tList: + t = "list"; + break; + case tLambda: + case tPrimOp: + case tPrimOpApp: + t = "lambda"; + break; + case tFloat: + t = "float"; + break; + default: + abort(); + } + mkString(v, state.symbols.Create(t)); +} + +/* Determine whether the argument is the null value. */ +static void prim_isNull(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tNull); +} + +/* Determine whether the argument is a function. */ +static void prim_isFunction(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + bool res; + switch (args[0]->type) { + case tLambda: + case tPrimOp: + case tPrimOpApp: + res = true; + break; + default: + res = false; + break; + } + mkBool(v, res); +} + +/* Determine whether the argument is an integer. */ +static void prim_isInt(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tInt); +} + +/* Determine whether the argument is a float. */ +static void prim_isFloat(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tFloat); +} + +/* Determine whether the argument is a string. */ +static void prim_isString(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tString); +} + +/* Determine whether the argument is a Boolean. */ +static void prim_isBool(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tBool); +} + +/* Determine whether the argument is a path. */ +static void prim_isPath(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tPath); +} + +struct CompareValues { + bool operator()(const Value* v1, const Value* v2) const { + if (v1->type == tFloat && v2->type == tInt) { + return v1->fpoint < v2->integer; + } + if (v1->type == tInt && v2->type == tFloat) { + return v1->integer < v2->fpoint; + } + if (v1->type != v2->type) { + throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % + showType(*v2)); + } + switch (v1->type) { + case tInt: + return v1->integer < v2->integer; + case tFloat: + return v1->fpoint < v2->fpoint; + case tString: + return strcmp(v1->string.s, v2->string.s) < 0; + case tPath: + return strcmp(v1->path, v2->path) < 0; + default: + throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % + showType(*v2)); + } + } +}; + +typedef std::list<Value*> ValueList; + +static void prim_genericClosure(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); + + /* Get the start set. */ + Bindings::iterator startSet = + args[0]->attrs->find(state.symbols.Create("startSet")); + if (startSet == args[0]->attrs->end()) { + throw EvalError(format("attribute 'startSet' required, at %1%") % pos); + } + state.forceList(*startSet->second.value, pos); + + ValueList workSet; + for (Value* elem : *startSet->second.value->list) { + workSet.push_back(elem); + } + + /* Get the operator. */ + Bindings::iterator op = + args[0]->attrs->find(state.symbols.Create("operator")); + if (op == args[0]->attrs->end()) { + throw EvalError(format("attribute 'operator' required, at %1%") % pos); + } + state.forceValue(*op->second.value); + + /* Construct the closure by applying the operator to element of + `workSet', adding the result to `workSet', continuing until + no new elements are found. */ + ValueList res; + // `doneKeys' doesn't need to be a GC root, because its values are + // reachable from res. + std::set<Value*, CompareValues> doneKeys; + while (!workSet.empty()) { + Value* e = *(workSet.begin()); + workSet.pop_front(); + + state.forceAttrs(*e, pos); + + Bindings::iterator key = e->attrs->find(state.symbols.Create("key")); + if (key == e->attrs->end()) { + throw EvalError(format("attribute 'key' required, at %1%") % pos); + } + state.forceValue(*key->second.value); + + if (doneKeys.find(key->second.value) != doneKeys.end()) { + continue; + } + doneKeys.insert(key->second.value); + res.push_back(e); + + /* Call the `operator' function with `e' as argument. */ + Value call; + mkApp(call, *op->second.value, *e); + state.forceList(call, pos); + + /* Add the values returned by the operator to the work set. */ + for (unsigned int n = 0; n < call.listSize(); ++n) { + state.forceValue(*(*call.list)[n]); + workSet.push_back((*call.list)[n]); + } + } + + /* Create the result list. */ + state.mkList(v, res.size()); + unsigned int n = 0; + for (auto& i : res) { + (*v.list)[n++] = i; + } +} + +static void prim_abort(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + std::string s = state.coerceToString(pos, *args[0], context); + throw Abort( + format("evaluation aborted with the following error message: '%1%'") % s); +} + +static void prim_throw(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + std::string s = state.coerceToString(pos, *args[0], context); + throw ThrownError(s); +} + +static void prim_addErrorContext(EvalState& state, const Pos& pos, Value** args, + Value& v) { + try { + state.forceValue(*args[1]); + v = *args[1]; + } catch (Error& e) { + PathSet context; + e.addPrefix(format("%1%\n") % state.coerceToString(pos, *args[0], context)); + throw; + } +} + +/* Try evaluating the argument. Success => {success=true; value=something;}, + * else => {success=false; value=false;} */ +static void prim_tryEval(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.mkAttrs(v, 2); + try { + state.forceValue(*args[0]); + v.attrs->push_back(Attr(state.sValue, args[0])); + mkBool(*state.allocAttr(v, state.symbols.Create("success")), true); + } catch (AssertionError& e) { + mkBool(*state.allocAttr(v, state.sValue), false); + mkBool(*state.allocAttr(v, state.symbols.Create("success")), false); + } +} + +/* Return an environment variable. Use with care. */ +static void prim_getEnv(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string name = state.forceStringNoCtx(*args[0], pos); + mkString(v, evalSettings.restrictEval || evalSettings.pureEval + ? "" + : getEnv(name).value_or("")); +} + +/* Evaluate the first argument, then return the second argument. */ +static void prim_seq(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0]); + state.forceValue(*args[1]); + v = *args[1]; +} + +/* Evaluate the first argument deeply (i.e. recursing into lists and + attrsets), then return the second argument. */ +static void prim_deepSeq(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValueDeep(*args[0]); + state.forceValue(*args[1]); + v = *args[1]; +} + +/* Evaluate the first expression and print it on standard error. Then + return the second expression. Useful for debugging. */ +static void prim_trace(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + if (args[0]->type == tString) { + LOG(INFO) << "trace: " << args[0]->string.s; + } else { + LOG(INFO) << "trace: " << *args[0]; + } + state.forceValue(*args[1]); + v = *args[1]; +} + +void prim_valueSize(EvalState& state, const Pos& pos, Value** args, Value& v) { + /* We're not forcing the argument on purpose. */ + mkInt(v, valueSize(*args[0])); +} + +/************************************************************* + * Derivations + *************************************************************/ + +/* Construct (as a unobservable side effect) a Nix derivation + expression that performs the derivation described by the argument + set. Returns the original set extended with the following + attributes: `outPath' containing the primary output path of the + derivation; `drvPath' containing the path of the Nix expression; + and `type' set to `derivation' to indicate that this is a + derivation. */ +static void prim_derivationStrict(EvalState& state, const Pos& pos, + Value** args, Value& v) { + state.forceAttrs(*args[0], pos); + + /* Figure out the name first (for stack backtraces). */ + Bindings::iterator attr = args[0]->attrs->find(state.sName); + if (attr == args[0]->attrs->end()) { + throw EvalError(format("required attribute 'name' missing, at %1%") % pos); + } + std::string drvName; + Pos& posDrvName(*attr->second.pos); + try { + drvName = state.forceStringNoCtx(*attr->second.value, pos); + } catch (Error& e) { + e.addPrefix( + format("while evaluating the derivation attribute 'name' at %1%:\n") % + posDrvName); + throw; + } + + /* Check whether attributes should be passed as a JSON file. */ + std::ostringstream jsonBuf; + std::unique_ptr<JSONObject> jsonObject; + attr = args[0]->attrs->find(state.sStructuredAttrs); + if (attr != args[0]->attrs->end() && + state.forceBool(*attr->second.value, pos)) { + jsonObject = std::make_unique<JSONObject>(jsonBuf); + } + + /* Check whether null attributes should be ignored. */ + bool ignoreNulls = false; + attr = args[0]->attrs->find(state.sIgnoreNulls); + if (attr != args[0]->attrs->end()) { + ignoreNulls = state.forceBool(*attr->second.value, pos); + } + + /* Build the derivation expression by processing the attributes. */ + Derivation drv; + + PathSet context; + + std::optional<std::string> outputHash; + std::string outputHashAlgo; + bool outputHashRecursive = false; + + StringSet outputs; + outputs.insert("out"); + + for (auto& [_, i] : *args[0]->attrs) { + if (i.name == state.sIgnoreNulls) { + continue; + } + const std::string& key = i.name; + + auto handleHashMode = [&](const std::string& s) { + if (s == "recursive") { + outputHashRecursive = true; + } else if (s == "flat") { + outputHashRecursive = false; + } else { + throw EvalError( + "invalid value '%s' for 'outputHashMode' attribute, at %s", s, + posDrvName); + } + }; + + auto handleOutputs = [&](const Strings& ss) { + outputs.clear(); + for (auto& j : ss) { + if (outputs.find(j) != outputs.end()) { + throw EvalError(format("duplicate derivation output '%1%', at %2%") % + j % posDrvName); + } + /* !!! Check whether j is a valid attribute + name. */ + /* Derivations cannot be named ‘drv’, because + then we'd have an attribute ‘drvPath’ in + the resulting set. */ + if (j == "drv") { + throw EvalError( + format("invalid derivation output name 'drv', at %1%") % + posDrvName); + } + outputs.insert(j); + } + if (outputs.empty()) { + throw EvalError( + format("derivation cannot have an empty set of outputs, at %1%") % + posDrvName); + } + }; + + try { + if (ignoreNulls) { + state.forceValue(*i.value); + if (i.value->type == tNull) { + continue; + } + } + + /* The `args' attribute is special: it supplies the + command-line arguments to the builder. */ + if (i.name == state.sArgs) { + state.forceList(*i.value, pos); + for (unsigned int n = 0; n < i.value->listSize(); ++n) { + std::string s = state.coerceToString(posDrvName, *(*i.value->list)[n], + context, true); + drv.args.push_back(s); + } + } + + /* All other attributes are passed to the builder through + the environment. */ + else { + if (jsonObject) { + if (i.name == state.sStructuredAttrs) { + continue; + } + + auto placeholder(jsonObject->placeholder(key)); + printValueAsJSON(state, true, *i.value, placeholder, context); + + if (i.name == state.sBuilder) { + drv.builder = state.forceString(*i.value, context, posDrvName); + } else if (i.name == state.sSystem) { + drv.platform = state.forceStringNoCtx(*i.value, posDrvName); + } else if (i.name == state.sOutputHash) { + outputHash = state.forceStringNoCtx(*i.value, posDrvName); + } else if (i.name == state.sOutputHashAlgo) { + outputHashAlgo = state.forceStringNoCtx(*i.value, posDrvName); + } else if (i.name == state.sOutputHashMode) { + handleHashMode(state.forceStringNoCtx(*i.value, posDrvName)); + } else if (i.name == state.sOutputs) { + /* Require ‘outputs’ to be a list of strings. */ + state.forceList(*i.value, posDrvName); + Strings ss; + for (unsigned int n = 0; n < i.value->listSize(); ++n) { + ss.emplace_back( + state.forceStringNoCtx(*(*i.value->list)[n], posDrvName)); + } + handleOutputs(ss); + } + + } else { + auto s = state.coerceToString(posDrvName, *i.value, context, true); + drv.env.emplace(key, s); + if (i.name == state.sBuilder) { + drv.builder = s; + } else if (i.name == state.sSystem) { + drv.platform = s; + } else if (i.name == state.sOutputHash) { + outputHash = s; + } else if (i.name == state.sOutputHashAlgo) { + outputHashAlgo = s; + } else if (i.name == state.sOutputHashMode) { + handleHashMode(s); + } else if (i.name == state.sOutputs) { + handleOutputs(absl::StrSplit(s, absl::ByAnyChar(" \t\n\r"), + absl::SkipEmpty())); + } + } + } + + } catch (Error& e) { + e.addPrefix(format("while evaluating the attribute '%1%' of the " + "derivation '%2%' at %3%:\n") % + key % drvName % posDrvName); + throw; + } + } + + if (jsonObject) { + jsonObject.reset(); + drv.env.emplace("__json", jsonBuf.str()); + } + + /* Everything in the context of the strings in the derivation + attributes should be added as dependencies of the resulting + derivation. */ + for (auto& path : context) { + /* Paths marked with `=' denote that the path of a derivation + is explicitly passed to the builder. Since that allows the + builder to gain access to every path in the dependency + graph of the derivation (including all outputs), all paths + in the graph must be added to this derivation's list of + inputs to ensure that they are available when the builder + runs. */ + if (path.at(0) == '=') { + /* !!! This doesn't work if readOnlyMode is set. */ + PathSet refs; + state.store->computeFSClosure(std::string(path, 1), refs); + for (auto& j : refs) { + drv.inputSrcs.insert(j); + if (isDerivation(j)) { + drv.inputDrvs[j] = state.store->queryDerivationOutputNames(j); + } + } + } + + /* Handle derivation outputs of the form ‘!<name>!<path>’. */ + else if (path.at(0) == '!') { + std::pair<std::string, std::string> ctx = decodeContext(path); + drv.inputDrvs[ctx.first].insert(ctx.second); + } + + /* Otherwise it's a source file. */ + else { + drv.inputSrcs.insert(path); + } + } + + /* Do we have all required attributes? */ + if (drv.builder.empty()) { + throw EvalError(format("required attribute 'builder' missing, at %1%") % + posDrvName); + } + if (drv.platform.empty()) { + throw EvalError(format("required attribute 'system' missing, at %1%") % + posDrvName); + } + + /* Check whether the derivation name is valid. */ + checkStoreName(drvName); + if (isDerivation(drvName)) { + throw EvalError( + format("derivation names are not allowed to end in '%1%', at %2%") % + drvExtension % posDrvName); + } + + if (outputHash) { + /* Handle fixed-output derivations. */ + if (outputs.size() != 1 || *(outputs.begin()) != "out") { + throw Error(format("multiple outputs are not supported in fixed-output " + "derivations, at %1%") % + posDrvName); + } + + HashType ht = + outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); + auto hash_ = Hash::deserialize(*outputHash, ht); + auto h = Hash::unwrap_throw(hash_); + + Path outPath = + state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); + if (!jsonObject) { + drv.env["out"] = outPath; + } + drv.outputs["out"] = DerivationOutput( + outPath, (outputHashRecursive ? "r:" : "") + printHashType(h.type), + h.to_string(Base16, false)); + } + + else { + /* Construct the "masked" store derivation, which is the final + one except that in the list of outputs, the output paths + are empty, and the corresponding environment variables have + an empty value. This ensures that changes in the set of + output names do get reflected in the hash. */ + for (auto& i : outputs) { + if (!jsonObject) { + drv.env[i] = ""; + } + drv.outputs[i] = DerivationOutput("", "", ""); + } + + /* Use the masked derivation expression to compute the output + path. */ + Hash h = hashDerivationModulo(*state.store, drv); + + for (auto& i : drv.outputs) { + if (i.second.path.empty()) { + Path outPath = state.store->makeOutputPath(i.first, h, drvName); + if (!jsonObject) { + drv.env[i.first] = outPath; + } + i.second.path = outPath; + } + } + } + + /* Write the resulting term into the Nix store directory. */ + Path drvPath = writeDerivation(state.store, drv, drvName, state.repair); + + VLOG(2) << "instantiated '" << drvName << "' -> '" << drvPath << "'"; + + /* Optimisation, but required in read-only mode! because in that + case we don't actually write store derivations, so we can't + read them later. */ + drvHashes[drvPath] = hashDerivationModulo(*state.store, drv); + + state.mkAttrs(v, 1 + drv.outputs.size()); + mkString(*state.allocAttr(v, state.sDrvPath), drvPath, {"=" + drvPath}); + for (auto& i : drv.outputs) { + mkString(*state.allocAttr(v, state.symbols.Create(i.first)), i.second.path, + {"!" + i.first + "!" + drvPath}); + } +} + +/* Return a placeholder string for the specified output that will be + substituted by the corresponding output path at build time. For + example, 'placeholder "out"' returns the string + /1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9. At build + time, any occurence of this string in an derivation attribute will + be replaced with the concrete path in the Nix store of the output + ‘out’. */ +static void prim_placeholder(EvalState& state, const Pos& pos, Value** args, + Value& v) { + mkString(v, hashPlaceholder(state.forceStringNoCtx(*args[0], pos))); +} + +/************************************************************* + * Paths + *************************************************************/ + +/* Convert the argument to a path. !!! obsolete? */ +static void prim_toPath(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + mkString(v, canonPath(path), context); +} + +/* Allow a valid store path to be used in an expression. This is + useful in some generated expressions such as in nix-push, which + generates a call to a function with an already existing store path + as argument. You don't want to use `toPath' here because it copies + the path to the Nix store, which yields a copy like + /nix/store/newhash-oldhash-oldname. In the past, `toPath' had + special case behaviour for store paths, but that created weird + corner cases. */ +static void prim_storePath(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context)); + /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink + directly in the store. The latter condition is necessary so + e.g. nix-push does the right thing. */ + if (!state.store->isStorePath(path)) { + path = canonPath(path, true); + } + if (!state.store->isInStore(path)) { + throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % + path % pos); + } + Path path2 = state.store->toStorePath(path); + if (!settings.readOnlyMode) { + state.store->ensurePath(path2); + } + context.insert(path2); + mkString(v, path, context); +} + +static void prim_pathExists(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + try { + state.realiseContext(context); + } catch (InvalidPathError& e) { + throw EvalError(format("cannot check the existence of '%1%', since path " + "'%2%' is not valid, at %3%") % + path % e.path % pos); + } + + try { + mkBool(v, pathExists(state.checkSourcePath(path))); + } catch (SysError& e) { + /* Don't give away info from errors while canonicalising + ‘path’ in restricted mode. */ + mkBool(v, false); + } catch (RestrictedPathError& e) { + mkBool(v, false); + } +} + +/* Return the base name of the given string, i.e., everything + following the last slash. */ +static void prim_baseNameOf(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + mkString( + v, baseNameOf(state.coerceToString(pos, *args[0], context, false, false)), + context); +} + +/* Return the directory of the given path, i.e., everything before the + last slash. Return either a path or a string depending on the type + of the argument. */ +static void prim_dirOf(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); + if (args[0]->type == tPath) { + mkPath(v, dir.c_str()); + } else { + mkString(v, dir, context); + } +} + +/* Return the contents of a file as a string. */ +static void prim_readFile(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + try { + state.realiseContext(context); + } catch (InvalidPathError& e) { + throw EvalError( + format("cannot read '%1%', since path '%2%' is not valid, at %3%") % + path % e.path % pos); + } + std::string s = + readFile(state.checkSourcePath(state.toRealPath(path, context))); + if (s.find(static_cast<char>(0)) != std::string::npos) { + throw Error(format("the contents of the file '%1%' cannot be represented " + "as a Nix string") % + path); + } + mkString(v, s.c_str()); +} + +/* Find a file in the Nix search path. Used to implement <x> paths, + which are desugared to 'findFile __nixPath "x"'. */ +static void prim_findFile(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + + SearchPath searchPath; + + for (unsigned int n = 0; n < args[0]->listSize(); ++n) { + Value& v2(*(*args[0]->list)[n]); + state.forceAttrs(v2, pos); + + std::string prefix; + Bindings::iterator i = v2.attrs->find(state.symbols.Create("prefix")); + if (i != v2.attrs->end()) { + prefix = state.forceStringNoCtx(*i->second.value, pos); + } + + i = v2.attrs->find(state.symbols.Create("path")); + if (i == v2.attrs->end()) { + throw EvalError(format("attribute 'path' missing, at %1%") % pos); + } + + PathSet context; + std::string path = + state.coerceToString(pos, *i->second.value, context, false, false); + + try { + state.realiseContext(context); + } catch (InvalidPathError& e) { + throw EvalError( + format("cannot find '%1%', since path '%2%' is not valid, at %3%") % + path % e.path % pos); + } + + searchPath.emplace_back(prefix, path); + } + + std::string path = state.forceStringNoCtx(*args[1], pos); + + mkPath(v, + state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); +} + +/* Return the cryptographic hash of a file in base-16. */ +static void prim_hashFile(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string type = state.forceStringNoCtx(*args[0], pos); + HashType ht = parseHashType(type); + if (ht == htUnknown) { + throw Error(format("unknown hash type '%1%', at %2%") % type % pos); + } + + PathSet context; // discarded + Path p = state.coerceToPath(pos, *args[1], context); + + mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), + context); +} + +/* Read a directory (without . or ..) */ +static void prim_readDir(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet ctx; + Path path = state.coerceToPath(pos, *args[0], ctx); + try { + state.realiseContext(ctx); + } catch (InvalidPathError& e) { + throw EvalError( + format("cannot read '%1%', since path '%2%' is not valid, at %3%") % + path % e.path % pos); + } + + DirEntries entries = readDirectory(state.checkSourcePath(path)); + state.mkAttrs(v, entries.size()); + + for (auto& ent : entries) { + Value* ent_val = state.allocAttr(v, state.symbols.Create(ent.name)); + if (ent.type == DT_UNKNOWN) { + ent.type = getFileType(path + "/" + ent.name); + } + mkStringNoCopy(*ent_val, ent.type == DT_REG ? "regular" + : ent.type == DT_DIR ? "directory" + : ent.type == DT_LNK ? "symlink" + : "unknown"); + } +} + +/************************************************************* + * Creating files + *************************************************************/ + +/* Convert the argument (which can be any Nix expression) to an XML + representation returned in a string. Not all Nix expressions can + be sensibly or completely represented (e.g., functions). */ +static void prim_toXML(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::ostringstream out; + PathSet context; + printValueAsXML(state, true, false, *args[0], out, context); + mkString(v, out.str(), context); +} + +/* Convert the argument (which can be any Nix expression) to a JSON + string. Not all Nix expressions can be sensibly or completely + represented (e.g., functions). */ +static void prim_toJSON(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::ostringstream out; + PathSet context; + printValueAsJSON(state, true, *args[0], out, context); + mkString(v, out.str(), context); +} + +/* Parse a JSON string to a value. */ +static void prim_fromJSON(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string s = state.forceStringNoCtx(*args[0], pos); + parseJSON(state, s, v); +} + +/* Store a string in the Nix store as a source file that can be used + as an input by derivations. */ +static void prim_toFile(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + std::string name = state.forceStringNoCtx(*args[0], pos); + std::string contents = state.forceString(*args[1], context, pos); + + PathSet refs; + + for (auto path : context) { + if (path.at(0) != '/') { + throw EvalError(format("in 'toFile': the file '%1%' cannot refer to " + "derivation outputs, at %2%") % + name % pos); + } + refs.insert(path); + } + + Path storePath = + settings.readOnlyMode + ? state.store->computeStorePathForText(name, contents, refs) + : state.store->addTextToStore(name, contents, refs, state.repair); + + /* Note: we don't need to add `context' to the context of the + result, since `storePath' itself has references to the paths + used in args[1]. */ + + mkString(v, storePath, {storePath}); +} + +static void addPath(EvalState& state, const Pos& pos, const std::string& name, + const Path& path_, Value* filterFun, bool recursive, + const Hash& expectedHash, Value& v) { + const auto path = evalSettings.pureEval && expectedHash + ? path_ + : state.checkSourcePath(path_); + PathFilter filter = filterFun != nullptr ? ([&](const Path& path) { + auto st = lstat(path); + + /* Call the filter function. The first argument is the path, + the second is a string indicating the type of the file. */ + Value arg1; + mkString(arg1, path); + + Value fun2; + state.callFunction(*filterFun, arg1, fun2, noPos); + + Value arg2; + mkString(arg2, S_ISREG(st.st_mode) ? "regular" + : S_ISDIR(st.st_mode) ? "directory" + : S_ISLNK(st.st_mode) + ? "symlink" + : "unknown" /* not supported, will fail! */); + + Value res; + state.callFunction(fun2, arg2, res, noPos); + + return state.forceBool(res, pos); + }) + : defaultPathFilter; + + Path expectedStorePath; + if (expectedHash) { + expectedStorePath = + state.store->makeFixedOutputPath(recursive, expectedHash, name); + } + Path dstPath; + if (!expectedHash || !state.store->isValidPath(expectedStorePath)) { + dstPath = settings.readOnlyMode + ? state.store + ->computeStorePathForPath(name, path, recursive, + htSHA256, filter) + .first + : state.store->addToStore(name, path, recursive, htSHA256, + filter, state.repair); + if (expectedHash && expectedStorePath != dstPath) { + throw Error(format("store path mismatch in (possibly filtered) path " + "added from '%1%'") % + path); + } + } else { + dstPath = expectedStorePath; + } + + mkString(v, dstPath, {dstPath}); +} + +static void prim_filterSource(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + if (!context.empty()) { + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % + path % pos); + } + + state.forceValue(*args[0]); + if (args[0]->type != tLambda) { + throw TypeError(format("first argument in call to 'filterSource' is not a " + "function but %1%, at %2%") % + showType(*args[0]) % pos); + } + + addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v); +} + +static void prim_path(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); + Path path; + std::string name; + Value* filterFun = nullptr; + auto recursive = true; + Hash expectedHash; + + for (auto& attr : *args[0]->attrs) { + const std::string& n(attr.second.name); + if (n == "path") { + PathSet context; + path = state.coerceToPath(*attr.second.pos, *attr.second.value, context); + if (!context.empty()) { + throw EvalError( + format("string '%1%' cannot refer to other paths, at %2%") % path % + *attr.second.pos); + } + } else if (attr.second.name == state.sName) { + name = state.forceStringNoCtx(*attr.second.value, *attr.second.pos); + } else if (n == "filter") { + state.forceValue(*attr.second.value); + filterFun = attr.second.value; + } else if (n == "recursive") { + recursive = state.forceBool(*attr.second.value, *attr.second.pos); + } else if (n == "sha256") { + auto hash_ = Hash::deserialize( + state.forceStringNoCtx(*attr.second.value, *attr.second.pos), + htSHA256); + expectedHash = Hash::unwrap_throw(hash_); + } else { + throw EvalError( + format("unsupported argument '%1%' to 'addPath', at %2%") % + attr.second.name % *attr.second.pos); + } + } + if (path.empty()) { + throw EvalError(format("'path' required, at %1%") % pos); + } + if (name.empty()) { + name = baseNameOf(path); + } + + addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); +} + +/************************************************************* + * Sets + *************************************************************/ + +/* Return the names of the attributes in a set as a sorted list of + strings. */ +static void prim_attrNames(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); + + state.mkList(v, args[0]->attrs->size()); + + unsigned int n = 0; + for (auto& [key, value] : *args[0]->attrs) { + mkString(*((*v.list)[n++] = state.allocValue()), key); + } +} + +/* Return the values of the attributes in a set as a list, in the same + order as attrNames. */ +static void prim_attrValues(EvalState& state, const Pos& pos, Value** input, + Value& output) { + state.forceAttrs(*input[0], pos); + + state.mkList(output, input[0]->attrs->size()); + + unsigned int n = 0; + for (auto& [key, value] : *input[0]->attrs) { + (*output.list)[n++] = value.value; + } +} + +/* Dynamic version of the `.' operator. */ +void prim_getAttr(EvalState& state, const Pos& pos, Value** args, Value& v) { + std::string attr = state.forceStringNoCtx(*args[0], pos); + state.forceAttrs(*args[1], pos); + // !!! Should we create a symbol here or just do a lookup? + Bindings::iterator i = args[1]->attrs->find(state.symbols.Create(attr)); + if (i == args[1]->attrs->end()) { + throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos); + } + // !!! add to stack trace? + if (state.countCalls && (i->second.pos != nullptr)) { + state.attrSelects[*i->second.pos]++; + } + state.forceValue(*i->second.value); + v = *i->second.value; +} + +/* Return position information of the specified attribute. */ +void prim_unsafeGetAttrPos(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string attr = state.forceStringNoCtx(*args[0], pos); + state.forceAttrs(*args[1], pos); + Bindings::iterator i = args[1]->attrs->find(state.symbols.Create(attr)); + if (i == args[1]->attrs->end()) { + mkNull(v); + } else { + state.mkPos(v, i->second.pos); + } +} + +/* Dynamic version of the `?' operator. */ +static void prim_hasAttr(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string attr = state.forceStringNoCtx(*args[0], pos); + state.forceAttrs(*args[1], pos); + mkBool(v, args[1]->attrs->find(state.symbols.Create(attr)) != + args[1]->attrs->end()); +} + +/* Determine whether the argument is a set. */ +static void prim_isAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tAttrs); +} + +static void prim_removeAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); + state.forceList(*args[1], pos); + + /* Get the attribute names to be removed. */ + std::set<Symbol> names; + for (unsigned int i = 0; i < args[1]->listSize(); ++i) { + state.forceStringNoCtx(*(*args[1]->list)[i], pos); + names.insert(state.symbols.Create((*args[1]->list)[i]->string.s)); + } + + /* Copy all attributes not in that set. Note that we don't need + to sort v.attrs because it's a subset of an already sorted + vector. */ + state.mkAttrs(v, args[0]->attrs->size()); + for (auto& i : *args[0]->attrs) { + if (names.find(i.second.name) == names.end()) { + v.attrs->push_back(i.second); + } + } +} + +/* Builds a set from a list specifying (name, value) pairs. To be + precise, a list [{name = "name1"; value = value1;} ... {name = + "nameN"; value = valueN;}] is transformed to {name1 = value1; + ... nameN = valueN;}. In case of duplicate occurences of the same + name, the first takes precedence. */ +static void prim_listToAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + + state.mkAttrs(v, args[0]->listSize()); + + std::set<Symbol> seen; + + for (unsigned int i = 0; i < args[0]->listSize(); ++i) { + Value& v2(*(*args[0]->list)[i]); + state.forceAttrs(v2, pos); + + Bindings::iterator j = v2.attrs->find(state.sName); + if (j == v2.attrs->end()) { + throw TypeError( + format( + "'name' attribute missing in a call to 'listToAttrs', at %1%") % + pos); + } + std::string name = state.forceStringNoCtx(*j->second.value, pos); + + Symbol sym = state.symbols.Create(name); + if (seen.find(sym) == seen.end()) { + Bindings::iterator j2 = + // TODO(tazjin): this line used to construct the symbol again: + // state.symbols.Create(state.sValue)); + // Why? + v2.attrs->find(state.sValue); + if (j2 == v2.attrs->end()) { + throw TypeError(format("'value' attribute missing in a call to " + "'listToAttrs', at %1%") % + pos); + } + + v.attrs->push_back(Attr(sym, j2->second.value, j2->second.pos)); + seen.insert(sym); + } + } +} + +/* Return the right-biased intersection of two sets as1 and as2, + i.e. a set that contains every attribute from as2 that is also a + member of as1. */ +static void prim_intersectAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[1], pos); + + state.mkAttrs(v, std::min(args[0]->attrs->size(), args[1]->attrs->size())); + + for (auto& i : *args[0]->attrs) { + Bindings::iterator j = args[1]->attrs->find(i.second.name); + if (j != args[1]->attrs->end()) { + v.attrs->push_back(j->second); + } + } +} + +/* Collect each attribute named `attr' from a list of attribute sets. + Sets that don't contain the named attribute are ignored. + + Example: + catAttrs "a" [{a = 1;} {b = 0;} {a = 2;}] + => [1 2] +*/ +static void prim_catAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + Symbol attrName = state.symbols.Create(state.forceStringNoCtx(*args[0], pos)); + state.forceList(*args[1], pos); + + Value* res[args[1]->listSize()]; + unsigned int found = 0; + + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + Value& v2(*(*args[1]->list)[n]); + state.forceAttrs(v2, pos); + Bindings::iterator i = v2.attrs->find(attrName); + if (i != v2.attrs->end()) { + res[found++] = i->second.value; + } + } + + state.mkList(v, found); + for (unsigned int n = 0; n < found; ++n) { + (*v.list)[n] = res[n]; + } +} + +/* Return a set containing the names of the formal arguments expected + by the function `f'. The value of each attribute is a Boolean + denoting whether the corresponding argument has a default value. For + instance, + + functionArgs ({ x, y ? 123}: ...) + => { x = false; y = true; } + + "Formal argument" here refers to the attributes pattern-matched by + the function. Plain lambdas are not included, e.g. + + functionArgs (x: ...) + => { } +*/ +static void prim_functionArgs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + if (args[0]->type == tPrimOpApp || args[0]->type == tPrimOp) { + state.mkAttrs(v, 0); + return; + } + if (args[0]->type != tLambda) { + throw TypeError(format("'functionArgs' requires a function, at %1%") % pos); + } + + if (!args[0]->lambda.fun->matchAttrs) { + state.mkAttrs(v, 0); + return; + } + + state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size()); + for (auto& i : args[0]->lambda.fun->formals->formals) { + // !!! should optimise booleans (allocate only once) + // TODO(tazjin): figure out what the above comment means + mkBool(*state.allocAttr(v, i.name), i.def != nullptr); + } +} + +/* Apply a function to every element of an attribute set. */ +static void prim_mapAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[1], pos); + + state.mkAttrs(v, args[1]->attrs->size()); + + for (auto& i : *args[1]->attrs) { + Value* vName = state.allocValue(); + Value* vFun2 = state.allocValue(); + mkString(*vName, i.second.name); + mkApp(*vFun2, *args[0], *vName); + mkApp(*state.allocAttr(v, i.second.name), *vFun2, *i.second.value); + } +} + +/************************************************************* + * Lists + *************************************************************/ + +/* Determine whether the argument is a list. */ +static void prim_isList(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->isList()); +} + +static void elemAt(EvalState& state, const Pos& pos, Value& list, int n, + Value& v) { + state.forceList(list, pos); + if (n < 0 || static_cast<unsigned int>(n) >= list.listSize()) { + throw Error(format("list index %1% is out of bounds, at %2%") % n % pos); + } + state.forceValue(*(*list.list)[n]); + v = *(*list.list)[n]; +} + +/* Return the n-1'th element of a list. */ +static void prim_elemAt(EvalState& state, const Pos& pos, Value** args, + Value& v) { + elemAt(state, pos, *args[0], state.forceInt(*args[1], pos), v); +} + +/* Return the first element of a list. */ +static void prim_head(EvalState& state, const Pos& pos, Value** args, + Value& v) { + elemAt(state, pos, *args[0], 0, v); +} + +/* Return a list consisting of everything but the first element of + a list. Warning: this function takes O(n) time, so you probably + don't want to use it! */ +static void prim_tail(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + if (args[0]->listSize() == 0) { + throw Error(format("'tail' called on an empty list, at %1%") % pos); + } + state.mkList(v, args[0]->listSize() - 1); + for (unsigned int n = 0; n < v.listSize(); ++n) { + (*v.list)[n] = (*args[0]->list)[n + 1]; + } +} + +/* Apply a function to every element of a list. */ +static void prim_map(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceList(*args[1], pos); + + state.mkList(v, args[1]->listSize()); + + for (unsigned int n = 0; n < v.listSize(); ++n) { + mkApp(*((*v.list)[n] = state.allocValue()), *args[0], *(*args[1]->list)[n]); + } +} + +/* Filter a list using a predicate; that is, return a list containing + every element from the list for which the predicate function + returns true. */ +static void prim_filter(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + // FIXME: putting this on the stack is risky. + Value* vs[args[1]->listSize()]; + unsigned int k = 0; + + bool same = true; + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + Value res; + state.callFunction(*args[0], *(*args[1]->list)[n], res, noPos); + if (state.forceBool(res, pos)) { + vs[k++] = (*args[1]->list)[n]; + } else { + same = false; + } + } + + if (same) { + v = *args[1]; + } else { + state.mkList(v, k); + for (unsigned int n = 0; n < k; ++n) { + (*v.list)[n] = vs[n]; + } + } +} + +/* Return true if a list contains a given element. */ +static void prim_elem(EvalState& state, const Pos& pos, Value** args, + Value& v) { + bool res = false; + state.forceList(*args[1], pos); + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + if (state.eqValues(*args[0], *(*args[1]->list)[n])) { + res = true; + break; + } + } + mkBool(v, res); +} + +/* Concatenate a list of lists. */ +static void prim_concatLists(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + state.concatLists(v, *args[0]->list, pos); +} + +/* Return the length of a list. This is an O(1) time operation. */ +static void prim_length(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + mkInt(v, args[0]->listSize()); +} + +/* Reduce a list by applying a binary operator, from left to + right. The operator is applied strictly. */ +static void prim_foldlStrict(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[2], pos); + + if (args[2]->listSize() != 0u) { + Value* vCur = args[1]; + + for (unsigned int n = 0; n < args[2]->listSize(); ++n) { + Value vTmp; + state.callFunction(*args[0], *vCur, vTmp, pos); + vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); + state.callFunction(vTmp, *(*args[2]->list)[n], *vCur, pos); + } + state.forceValue(v); + } else { + state.forceValue(*args[1]); + v = *args[1]; + } +} + +static void anyOrAll(bool any, EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + Value vTmp; + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + state.callFunction(*args[0], *(*args[1]->list)[n], vTmp, pos); + bool res = state.forceBool(vTmp, pos); + if (res == any) { + mkBool(v, any); + return; + } + } + + mkBool(v, !any); +} + +static void prim_any(EvalState& state, const Pos& pos, Value** args, Value& v) { + anyOrAll(true, state, pos, args, v); +} + +static void prim_all(EvalState& state, const Pos& pos, Value** args, Value& v) { + anyOrAll(false, state, pos, args, v); +} + +static void prim_genList(EvalState& state, const Pos& pos, Value** args, + Value& v) { + auto len = state.forceInt(*args[1], pos); + + if (len < 0) { + throw EvalError(format("cannot create list of size %1%, at %2%") % len % + pos); + } + + state.mkList(v, len); + + for (unsigned int n = 0; n < static_cast<unsigned int>(len); ++n) { + Value* arg = state.allocValue(); + mkInt(*arg, n); + mkApp(*((*v.list)[n] = state.allocValue()), *args[0], *arg); + } +} + +static void prim_lessThan(EvalState& state, const Pos& pos, Value** args, + Value& v); + +static void prim_sort(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + // Copy of the input list which can be sorted in place. + v.type = tList; + v.list = std::make_shared<NixList>(*args[1]->list); + + std::for_each(v.list->begin(), v.list->end(), + [&](Value* val) { state.forceValue(*val); }); + + auto comparator = [&](Value* a, Value* b) { + /* Optimization: if the comparator is lessThan, bypass + callFunction. */ + if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan) { + return CompareValues()(a, b); + } + + Value vTmp1{}; + Value vTmp2{}; + state.callFunction(*args[0], *a, vTmp1, pos); + state.callFunction(vTmp1, *b, vTmp2, pos); + return state.forceBool(vTmp2, pos); + }; + + /* FIXME: std::sort can segfault if the comparator is not a strict + weak ordering. What to do? std::stable_sort() seems more + resilient, but no guarantees... */ + std::stable_sort(v.list->begin(), v.list->end(), comparator); +} + +static void prim_partition(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + std::shared_ptr<NixList> right = std::make_shared<NixList>(); + std::shared_ptr<NixList> wrong = std::make_shared<NixList>(); + + for (Value* elem : *args[1]->list) { + state.forceValue(*elem, pos); + + Value res; + state.callFunction(*args[0], *elem, res, pos); + if (state.forceBool(res, pos)) { + right->push_back(elem); + } else { + wrong->push_back(elem); + } + } + + state.mkAttrs(v, 2); + + Value* vRight = state.allocAttr(v, state.sRight); + state.mkList(*vRight, right); + + Value* vWrong = state.allocAttr(v, state.sWrong); + state.mkList(*vWrong, wrong); +} + +/* concatMap = f: list: concatLists (map f list); */ +/* C++-version is to avoid allocating `mkApp', call `f' eagerly */ +static void prim_concatMap(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + std::shared_ptr<NixList> outlist = std::make_shared<NixList>(); + + for (Value* elem : *args[1]->list) { + auto out = state.allocValue(); + state.callFunction(*args[0], *elem, *out, pos); + state.forceList(*out, pos); + + outlist->insert(outlist->end(), out->list->begin(), out->list->end()); + } + + state.mkList(v, outlist); +} + +/************************************************************* + * Integer arithmetic + *************************************************************/ + +static void prim_add(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + if (args[0]->type == tFloat || args[1]->type == tFloat) { + mkFloat(v, + state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); + } else { + mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); + } +} + +static void prim_sub(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + if (args[0]->type == tFloat || args[1]->type == tFloat) { + mkFloat(v, + state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); + } else { + mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); + } +} + +static void prim_mul(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + if (args[0]->type == tFloat || args[1]->type == tFloat) { + mkFloat(v, + state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); + } else { + mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); + } +} + +static void prim_div(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + + NixFloat f2 = state.forceFloat(*args[1], pos); + if (f2 == 0) { + throw EvalError(format("division by zero, at %1%") % pos); + } + + if (args[0]->type == tFloat || args[1]->type == tFloat) { + mkFloat(v, + state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); + } else { + NixInt i1 = state.forceInt(*args[0], pos); + NixInt i2 = state.forceInt(*args[1], pos); + /* Avoid division overflow as it might raise SIGFPE. */ + if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1) { + throw EvalError(format("overflow in integer division, at %1%") % pos); + } + mkInt(v, i1 / i2); + } +} + +static void prim_bitAnd(EvalState& state, const Pos& pos, Value** args, + Value& v) { + mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); +} + +static void prim_bitOr(EvalState& state, const Pos& pos, Value** args, + Value& v) { + mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); +} + +static void prim_bitXor(EvalState& state, const Pos& pos, Value** args, + Value& v) { + mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); +} + +static void prim_lessThan(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + state.forceValue(*args[1]); + CompareValues comp; + mkBool(v, comp(args[0], args[1])); +} + +/************************************************************* + * String manipulation + *************************************************************/ + +/* Convert the argument to a string. Paths are *not* copied to the + store, so `toString /foo/bar' yields `"/foo/bar"', not + `"/nix/store/whatever..."'. */ +static void prim_toString(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + std::string s = state.coerceToString(pos, *args[0], context, true, false); + mkString(v, s, context); +} + +/* `substring start len str' returns the substring of `str' starting + at character position `min(start, stringLength str)' inclusive and + ending at `min(start + len, stringLength str)'. `start' must be + non-negative. */ +static void prim_substring(EvalState& state, const Pos& pos, Value** args, + Value& v) { + int start = state.forceInt(*args[0], pos); + int len = state.forceInt(*args[1], pos); + PathSet context; + std::string s = state.coerceToString(pos, *args[2], context); + + if (start < 0) { + throw EvalError(format("negative start position in 'substring', at %1%") % + pos); + } + + mkString(v, + static_cast<unsigned int>(start) >= s.size() + ? "" + : std::string(s, start, len), + context); +} + +static void prim_stringLength(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + std::string s = state.coerceToString(pos, *args[0], context); + mkInt(v, s.size()); +} + +/* Return the cryptographic hash of a string in base-16. */ +static void prim_hashString(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string type = state.forceStringNoCtx(*args[0], pos); + HashType ht = parseHashType(type); + if (ht == htUnknown) { + throw Error(format("unknown hash type '%1%', at %2%") % type % pos); + } + + PathSet context; // discarded + std::string s = state.forceString(*args[1], context, pos); + + mkString(v, hashString(ht, s).to_string(Base16, false), context); +} + +/* Match a regular expression against a string and return either + ‘null’ or a list containing substring matches. */ +static void prim_match(EvalState& state, const Pos& pos, Value** args, + Value& v) { + auto re = state.forceStringNoCtx(*args[0], pos); + + try { + std::regex regex(re, std::regex::extended); + + PathSet context; + const std::string str = state.forceString(*args[1], context, pos); + + std::smatch match; + if (!std::regex_match(str, match, regex)) { + mkNull(v); + return; + } + + // the first match is the whole string + const size_t len = match.size() - 1; + state.mkList(v, len); + for (size_t i = 0; i < len; ++i) { + if (!match[i + 1].matched) { + mkNull(*((*v.list)[i] = state.allocValue())); + } else { + mkString(*((*v.list)[i] = state.allocValue()), + match[i + 1].str().c_str()); + } + } + + } catch (std::regex_error& e) { + if (e.code() == std::regex_constants::error_space) { + // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ + throw EvalError("memory limit exceeded by regular expression '%s', at %s", + re, pos); + } + throw EvalError("invalid regular expression '%s', at %s", re, pos); + } +} + +/* Split a std::string with a regular expression, and return a list of the + non-matching parts interleaved by the lists of the matching groups. */ +static void prim_split(EvalState& state, const Pos& pos, Value** args, + Value& v) { + auto re = state.forceStringNoCtx(*args[0], pos); + + try { + std::regex regex(re, std::regex::extended); + + PathSet context; + const std::string str = state.forceString(*args[1], context, pos); + + auto begin = std::sregex_iterator(str.begin(), str.end(), regex); + auto end = std::sregex_iterator(); + + // Any matches results are surrounded by non-matching results. + const size_t len = std::distance(begin, end); + state.mkList(v, 2 * len + 1); + size_t idx = 0; + Value* elem; + + if (len == 0) { + (*v.list)[idx++] = args[1]; + return; + } + + for (std::sregex_iterator i = begin; i != end; ++i) { + assert(idx <= 2 * len + 1 - 3); + std::smatch match = *i; + + // Add a string for non-matched characters. + elem = (*v.list)[idx++] = state.allocValue(); + mkString(*elem, match.prefix().str().c_str()); + + // Add a list for matched substrings. + const size_t slen = match.size() - 1; + elem = (*v.list)[idx++] = state.allocValue(); + + // Start at 1, beacause the first match is the whole string. + state.mkList(*elem, slen); + for (size_t si = 0; si < slen; ++si) { + if (!match[si + 1].matched) { + mkNull(*((*elem->list)[si] = state.allocValue())); + } else { + mkString(*((*elem->list)[si] = state.allocValue()), + match[si + 1].str().c_str()); + } + } + + // Add a string for non-matched suffix characters. + if (idx == 2 * len) { + elem = (*v.list)[idx++] = state.allocValue(); + mkString(*elem, match.suffix().str().c_str()); + } + } + assert(idx == 2 * len + 1); + + } catch (std::regex_error& e) { + if (e.code() == std::regex_constants::error_space) { + // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ + throw EvalError("memory limit exceeded by regular expression '%s', at %s", + re, pos); + } + throw EvalError("invalid regular expression '%s', at %s", re, pos); + } +} + +static void prim_concatStringSep(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + + auto sep = state.forceString(*args[0], context, pos); + state.forceList(*args[1], pos); + + std::string res; + res.reserve((args[1]->listSize() + 32) * sep.size()); + bool first = true; + + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + if (first) { + first = false; + } else { + res += sep; + } + + res += state.coerceToString(pos, *(*args[1]->list)[n], context); + } + + mkString(v, res, context); +} + +static void prim_replaceStrings(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + state.forceList(*args[1], pos); + if (args[0]->listSize() != args[1]->listSize()) { + throw EvalError(format("'from' and 'to' arguments to 'replaceStrings' have " + "different lengths, at %1%") % + pos); + } + + std::vector<std::string> from; + from.reserve(args[0]->listSize()); + for (unsigned int n = 0; n < args[0]->listSize(); ++n) { + from.push_back(state.forceString(*(*args[0]->list)[n], pos)); + } + + std::vector<std::pair<std::string, PathSet>> to; + to.reserve(args[1]->listSize()); + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + PathSet ctx; + auto s = state.forceString(*(*args[1]->list)[n], ctx, pos); + to.emplace_back(std::move(s), std::move(ctx)); + } + + PathSet context; + auto s = state.forceString(*args[2], context, pos); + + std::string res; + // Loops one past last character to handle the case where 'from' contains an + // empty string. + for (size_t p = 0; p <= s.size();) { + bool found = false; + auto i = from.begin(); + auto j = to.begin(); + for (; i != from.end(); ++i, ++j) { + if (s.compare(p, i->size(), *i) == 0) { + found = true; + res += j->first; + if (i->empty()) { + if (p < s.size()) { + res += s[p]; + } + p++; + } else { + p += i->size(); + } + for (auto& path : j->second) { + context.insert(path); + } + j->second.clear(); + break; + } + } + if (!found) { + if (p < s.size()) { + res += s[p]; + } + p++; + } + } + + mkString(v, res, context); +} + +/************************************************************* + * Versions + *************************************************************/ + +static void prim_parseDrvName(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string name = state.forceStringNoCtx(*args[0], pos); + DrvName parsed(name); + state.mkAttrs(v, 2); + mkString(*state.allocAttr(v, state.sName), parsed.name); + mkString(*state.allocAttr(v, state.symbols.Create("version")), + parsed.version); +} + +static void prim_compareVersions(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string version1 = state.forceStringNoCtx(*args[0], pos); + std::string version2 = state.forceStringNoCtx(*args[1], pos); + mkInt(v, compareVersions(version1, version2)); +} + +static void prim_splitVersion(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string version = state.forceStringNoCtx(*args[0], pos); + auto iter = version.cbegin(); + Strings components; + while (iter != version.cend()) { + auto component = nextComponent(iter, version.cend()); + if (component.empty()) { + break; + } + components.emplace_back(std::move(component)); + } + state.mkList(v, components.size()); + unsigned int n = 0; + for (auto& component : components) { + auto listElem = (*v.list)[n++] = state.allocValue(); + mkString(*listElem, component); + } +} + +/************************************************************* + * Networking + *************************************************************/ + +void fetch(EvalState& state, const Pos& pos, Value** args, Value& v, + const std::string& who, bool unpack, + const std::string& defaultName) { + CachedDownloadRequest request(""); + request.unpack = unpack; + request.name = defaultName; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); + + for (auto& attr : *args[0]->attrs) { + std::string n(attr.second.name); + if (n == "url") { + request.uri = + state.forceStringNoCtx(*attr.second.value, *attr.second.pos); + } else if (n == "sha256") { + auto hash_ = Hash::deserialize( + state.forceStringNoCtx(*attr.second.value, *attr.second.pos), + htSHA256); + request.expectedHash = Hash::unwrap_throw(hash_); + } else if (n == "name") { + request.name = + state.forceStringNoCtx(*attr.second.value, *attr.second.pos); + } else { + throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % + attr.second.name % who % attr.second.pos); + } + } + + if (request.uri.empty()) { + throw EvalError(format("'url' argument required, at %1%") % pos); + } + + } else { + request.uri = state.forceStringNoCtx(*args[0], pos); + } + + state.checkURI(request.uri); + + if (evalSettings.pureEval && !request.expectedHash) { + throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", + who); + } + + auto res = getDownloader()->downloadCached(state.store, request); + + if (state.allowedPaths) { + state.allowedPaths->insert(res.path); + } + + mkString(v, res.storePath, PathSet({res.storePath})); +} + +static void prim_fetchurl(EvalState& state, const Pos& pos, Value** args, + Value& v) { + fetch(state, pos, args, v, "fetchurl", false, ""); +} + +static void prim_fetchTarball(EvalState& state, const Pos& pos, Value** args, + Value& v) { + fetch(state, pos, args, v, "fetchTarball", true, "source"); +} + +/************************************************************* + * Primop registration + *************************************************************/ + +RegisterPrimOp::PrimOps* RegisterPrimOp::primOps; + +RegisterPrimOp::RegisterPrimOp(const std::string& name, size_t arity, + PrimOpFun fun) { + if (primOps == nullptr) { + primOps = new PrimOps; + } + primOps->emplace_back(name, arity, fun); +} + +void EvalState::createBaseEnv() { + baseEnv.up = nullptr; + + /* Add global constants such as `true' to the base environment. */ + Value v; + + /* `builtins' must be first! */ + mkAttrs(v, 128); + addConstant("builtins", v); + + mkBool(v, true); + addConstant("true", v); + + mkBool(v, false); + addConstant("false", v); + + mkNull(v); + addConstant("null", v); + + auto vThrow = addPrimOp("throw", 1, prim_throw); + + auto addPurityError = [&](const std::string& name) { + Value* v2 = allocValue(); + mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name)); + mkApp(v, *vThrow, *v2); + addConstant(name, v); + }; + + if (!evalSettings.pureEval) { + mkInt(v, time(nullptr)); + addConstant("__currentTime", v); + } + + if (!evalSettings.pureEval) { + mkString(v, settings.thisSystem); + addConstant("__currentSystem", v); + } + + mkString(v, nixVersion); + addConstant("__nixVersion", v); + + mkString(v, store->storeDir); + addConstant("__storeDir", v); + + /* Language version. This should be increased every time a new + language feature gets added. It's not necessary to increase it + when primops get added, because you can just use `builtins ? + primOp' to check. */ + mkInt(v, 5); + addConstant("__langVersion", v); + + // Miscellaneous + auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport); + Value* v2 = allocValue(); + mkAttrs(*v2, 0); + mkApp(v, *vScopedImport, *v2); + forceValue(v); + addConstant("import", v); + addPrimOp("__typeOf", 1, prim_typeOf); + addPrimOp("isNull", 1, prim_isNull); + addPrimOp("__isFunction", 1, prim_isFunction); + addPrimOp("__isString", 1, prim_isString); + addPrimOp("__isInt", 1, prim_isInt); + addPrimOp("__isFloat", 1, prim_isFloat); + addPrimOp("__isBool", 1, prim_isBool); + addPrimOp("__isPath", 1, prim_isPath); + addPrimOp("__genericClosure", 1, prim_genericClosure); + addPrimOp("abort", 1, prim_abort); + addPrimOp("__addErrorContext", 2, prim_addErrorContext); + addPrimOp("__tryEval", 1, prim_tryEval); + addPrimOp("__getEnv", 1, prim_getEnv); + + // Strictness + addPrimOp("__seq", 2, prim_seq); + addPrimOp("__deepSeq", 2, prim_deepSeq); + + // Debugging + addPrimOp("__trace", 2, prim_trace); + addPrimOp("__valueSize", 1, prim_valueSize); + + // Paths + addPrimOp("__toPath", 1, prim_toPath); + if (evalSettings.pureEval) { + addPurityError("__storePath"); + } else { + addPrimOp("__storePath", 1, prim_storePath); + } + addPrimOp("__pathExists", 1, prim_pathExists); + addPrimOp("baseNameOf", 1, prim_baseNameOf); + addPrimOp("dirOf", 1, prim_dirOf); + addPrimOp("__readFile", 1, prim_readFile); + addPrimOp("__readDir", 1, prim_readDir); + addPrimOp("__findFile", 2, prim_findFile); + addPrimOp("__hashFile", 2, prim_hashFile); + + // Creating files + addPrimOp("__toXML", 1, prim_toXML); + addPrimOp("__toJSON", 1, prim_toJSON); + addPrimOp("__fromJSON", 1, prim_fromJSON); + addPrimOp("__toFile", 2, prim_toFile); + addPrimOp("__filterSource", 2, prim_filterSource); + addPrimOp("__path", 1, prim_path); + + // Sets + addPrimOp("__attrNames", 1, prim_attrNames); + addPrimOp("__attrValues", 1, prim_attrValues); + addPrimOp("__getAttr", 2, prim_getAttr); + addPrimOp("__unsafeGetAttrPos", 2, prim_unsafeGetAttrPos); + addPrimOp("__hasAttr", 2, prim_hasAttr); + addPrimOp("__isAttrs", 1, prim_isAttrs); + addPrimOp("removeAttrs", 2, prim_removeAttrs); + addPrimOp("__listToAttrs", 1, prim_listToAttrs); + addPrimOp("__intersectAttrs", 2, prim_intersectAttrs); + addPrimOp("__catAttrs", 2, prim_catAttrs); + addPrimOp("__functionArgs", 1, prim_functionArgs); + addPrimOp("__mapAttrs", 2, prim_mapAttrs); + + // Lists + addPrimOp("__isList", 1, prim_isList); + addPrimOp("__elemAt", 2, prim_elemAt); + addPrimOp("__head", 1, prim_head); + addPrimOp("__tail", 1, prim_tail); + addPrimOp("map", 2, prim_map); + addPrimOp("__filter", 2, prim_filter); + addPrimOp("__elem", 2, prim_elem); + addPrimOp("__concatLists", 1, prim_concatLists); + addPrimOp("__length", 1, prim_length); + addPrimOp("__foldl'", 3, prim_foldlStrict); + addPrimOp("__any", 2, prim_any); + addPrimOp("__all", 2, prim_all); + addPrimOp("__genList", 2, prim_genList); + addPrimOp("__sort", 2, prim_sort); + addPrimOp("__partition", 2, prim_partition); + addPrimOp("__concatMap", 2, prim_concatMap); + + // Integer arithmetic + addPrimOp("__add", 2, prim_add); + addPrimOp("__sub", 2, prim_sub); + addPrimOp("__mul", 2, prim_mul); + addPrimOp("__div", 2, prim_div); + addPrimOp("__bitAnd", 2, prim_bitAnd); + addPrimOp("__bitOr", 2, prim_bitOr); + addPrimOp("__bitXor", 2, prim_bitXor); + addPrimOp("__lessThan", 2, prim_lessThan); + + // String manipulation + addPrimOp("toString", 1, prim_toString); + addPrimOp("__substring", 3, prim_substring); + addPrimOp("__stringLength", 1, prim_stringLength); + addPrimOp("__hashString", 2, prim_hashString); + addPrimOp("__match", 2, prim_match); + addPrimOp("__split", 2, prim_split); + addPrimOp("__concatStringsSep", 2, prim_concatStringSep); + addPrimOp("__replaceStrings", 3, prim_replaceStrings); + + // Versions + addPrimOp("__parseDrvName", 1, prim_parseDrvName); + addPrimOp("__compareVersions", 2, prim_compareVersions); + addPrimOp("__splitVersion", 1, prim_splitVersion); + + // Derivations + addPrimOp("derivationStrict", 1, prim_derivationStrict); + addPrimOp("placeholder", 1, prim_placeholder); + + // Networking + addPrimOp("__fetchurl", 1, prim_fetchurl); + addPrimOp("fetchTarball", 1, prim_fetchTarball); + + /* Add a wrapper around the derivation primop that computes the + `drvPath' and `outPath' attributes lazily. */ + std::string path = + canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); + sDerivationNix = symbols.Create(path); + evalFile(path, v); + addConstant("derivation", v); + + /* Add a value containing the current Nix expression search path. */ + mkList(v, searchPath.size()); + int n = 0; + for (auto& i : searchPath) { + v2 = (*v.list)[n++] = allocValue(); + mkAttrs(*v2, 2); + mkString(*allocAttr(*v2, symbols.Create("path")), i.second); + mkString(*allocAttr(*v2, symbols.Create("prefix")), i.first); + } + addConstant("__nixPath", v); + + if (RegisterPrimOp::primOps != nullptr) { + for (auto& primOp : *RegisterPrimOp::primOps) { + addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp)); + } + } +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops.hh b/third_party/nix/src/libexpr/primops.hh new file mode 100644 index 0000000000..ab5f647202 --- /dev/null +++ b/third_party/nix/src/libexpr/primops.hh @@ -0,0 +1,17 @@ +#include <tuple> +#include <vector> + +#include "libexpr/eval.hh" + +namespace nix { + +struct RegisterPrimOp { + using PrimOps = std::vector<std::tuple<std::string, size_t, PrimOpFun> >; + static PrimOps* primOps; + /* You can register a constant by passing an arity of 0. fun + will get called during EvalState initialization, so there + may be primops not yet added and builtins is not yet sorted. */ + RegisterPrimOp(const std::string& name, size_t arity, PrimOpFun fun); +}; + +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops/context.cc b/third_party/nix/src/libexpr/primops/context.cc new file mode 100644 index 0000000000..fb8879ead1 --- /dev/null +++ b/third_party/nix/src/libexpr/primops/context.cc @@ -0,0 +1,202 @@ +#include "libexpr/eval-inline.hh" +#include "libexpr/primops.hh" +#include "libstore/derivations.hh" + +namespace nix { + +static void prim_unsafeDiscardStringContext(EvalState& state, const Pos& pos, + Value** args, Value& v) { + PathSet context; + std::string s = state.coerceToString(pos, *args[0], context); + mkString(v, s, PathSet()); +} + +static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, + prim_unsafeDiscardStringContext); + +static void prim_hasContext(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + state.forceString(*args[0], context, pos); + mkBool(v, !context.empty()); +} + +static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); + +/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a + builder without causing the derivation to be built (for instance, + in the derivation that builds NARs in nix-push, when doing + source-only deployment). This primop marks the string context so + that builtins.derivation adds the path to drv.inputSrcs rather than + drv.inputDrvs. */ +static void prim_unsafeDiscardOutputDependency(EvalState& state, const Pos& pos, + Value** args, Value& v) { + PathSet context; + std::string s = state.coerceToString(pos, *args[0], context); + + PathSet context2; + for (auto& p : context) { + context2.insert(p.at(0) == '=' ? std::string(p, 1) : p); + } + + mkString(v, s, context2); +} + +static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, + prim_unsafeDiscardOutputDependency); + +/* Extract the context of a string as a structured Nix value. + + The context is represented as an attribute set whose keys are the + paths in the context set and whose values are attribute sets with + the following keys: + path: True if the relevant path is in the context as a plain store + path (i.e. the kind of context you get when interpolating + a Nix path (e.g. ./.) into a string). False if missing. + allOutputs: True if the relevant path is a derivation and it is + in the context as a drv file with all of its outputs + (i.e. the kind of context you get when referencing + .drvPath of some derivation). False if missing. + outputs: If a non-empty list, the relevant path is a derivation + and the provided outputs are referenced in the context + (i.e. the kind of context you get when referencing + .outPath of some derivation). Empty list if missing. + Note that for a given path any combination of the above attributes + may be present. +*/ +static void prim_getContext(EvalState& state, const Pos& pos, Value** args, + Value& v) { + struct ContextInfo { + bool path = false; + bool allOutputs = false; + Strings outputs; + }; + PathSet context; + state.forceString(*args[0], context, pos); + auto contextInfos = std::map<Path, ContextInfo>(); + for (const auto& p : context) { + Path drv; + std::string output; + const Path* path = &p; + if (p.at(0) == '=') { + drv = std::string(p, 1); + path = &drv; + } else if (p.at(0) == '!') { + std::pair<std::string, std::string> ctx = decodeContext(p); + drv = ctx.first; + output = ctx.second; + path = &drv; + } + auto isPath = drv.empty(); + auto isAllOutputs = (!drv.empty()) && output.empty(); + + auto iter = contextInfos.find(*path); + if (iter == contextInfos.end()) { + contextInfos.emplace( + *path, + ContextInfo{isPath, isAllOutputs, + output.empty() ? Strings{} : Strings{std::move(output)}}); + } else { + if (isPath) { + iter->second.path = true; + } else if (isAllOutputs) { + iter->second.allOutputs = true; + } else { + iter->second.outputs.emplace_back(std::move(output)); + } + } + } + + state.mkAttrs(v, contextInfos.size()); + + auto sPath = state.symbols.Create("path"); + auto sAllOutputs = state.symbols.Create("allOutputs"); + for (const auto& info : contextInfos) { + auto& infoVal = *state.allocAttr(v, state.symbols.Create(info.first)); + state.mkAttrs(infoVal, 3); + if (info.second.path) { + mkBool(*state.allocAttr(infoVal, sPath), true); + } + if (info.second.allOutputs) { + mkBool(*state.allocAttr(infoVal, sAllOutputs), true); + } + if (!info.second.outputs.empty()) { + auto& outputsVal = *state.allocAttr(infoVal, state.sOutputs); + state.mkList(outputsVal, info.second.outputs.size()); + size_t i = 0; + for (const auto& output : info.second.outputs) { + mkString(*((*outputsVal.list)[i++] = state.allocValue()), output); + } + } + } +} + +static RegisterPrimOp r4("__getContext", 1, prim_getContext); + +/* Append the given context to a given string. + + See the commentary above unsafeGetContext for details of the + context representation. +*/ +static void prim_appendContext(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + auto orig = state.forceString(*args[0], context, pos); + + state.forceAttrs(*args[1], pos); + + auto sPath = state.symbols.Create("path"); + auto sAllOutputs = state.symbols.Create("allOutputs"); + for (const auto& attr_iter : *args[1]->attrs) { + const Attr* i = &attr_iter.second; // TODO(tazjin): get rid of this + if (!state.store->isStorePath(i->name)) { + throw EvalError("Context key '%s' is not a store path, at %s", i->name, + i->pos); + } + if (!settings.readOnlyMode) { + state.store->ensurePath(i->name); + } + state.forceAttrs(*i->value, *i->pos); + auto iter = i->value->attrs->find(sPath); + if (iter != i->value->attrs->end()) { + if (state.forceBool(*iter->second.value, *iter->second.pos)) { + context.insert(i->name); + } + } + + iter = i->value->attrs->find(sAllOutputs); + if (iter != i->value->attrs->end()) { + if (state.forceBool(*iter->second.value, *iter->second.pos)) { + if (!isDerivation(i->name)) { + throw EvalError( + "Tried to add all-outputs context of %s, which is not a " + "derivation, to a string, at %s", + i->name, i->pos); + } + context.insert("=" + std::string(i->name)); + } + } + + iter = i->value->attrs->find(state.sOutputs); + if (iter != i->value->attrs->end()) { + state.forceList(*iter->second.value, *iter->second.pos); + if (iter->second.value->listSize() && !isDerivation(i->name)) { + throw EvalError( + "Tried to add derivation output context of %s, which is not a " + "derivation, to a string, at %s", + i->name, i->pos); + } + for (unsigned int n = 0; n < iter->second.value->listSize(); ++n) { + auto name = state.forceStringNoCtx(*(*iter->second.value->list)[n], + *iter->second.pos); + context.insert("!" + name + "!" + std::string(i->name)); + } + } + } + + mkString(v, orig, context); +} + +static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops/fetchGit.cc b/third_party/nix/src/libexpr/primops/fetchGit.cc new file mode 100644 index 0000000000..da4d683401 --- /dev/null +++ b/third_party/nix/src/libexpr/primops/fetchGit.cc @@ -0,0 +1,277 @@ +#include <nlohmann/json.hpp> +#include <regex> + +#include <absl/strings/ascii.h> +#include <absl/strings/match.h> +#include <absl/strings/str_split.h> +#include <glog/logging.h> +#include <sys/time.h> + +#include "libexpr/eval-inline.hh" +#include "libexpr/primops.hh" +#include "libstore/download.hh" +#include "libstore/pathlocks.hh" +#include "libstore/store-api.hh" +#include "libutil/hash.hh" + +using namespace std::string_literals; + +namespace nix { + +struct GitInfo { + Path storePath; + std::string rev; + std::string shortRev; + uint64_t revCount = 0; +}; + +std::regex revRegex("^[0-9a-fA-F]{40}$"); + +GitInfo exportGit(ref<Store> store, const std::string& uri, + std::optional<std::string> ref, std::string rev, + const std::string& name) { + if (evalSettings.pureEval && rev == "") { + throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); + } + + if (!ref && rev == "" && absl::StartsWith(uri, "/") && + pathExists(uri + "/.git")) { + bool clean = true; + + try { + runProgram("git", true, + {"-C", uri, "diff-index", "--quiet", "HEAD", "--"}); + } catch (ExecError& e) { + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) { + throw; + } + clean = false; + } + + if (!clean) { + /* This is an unclean working tree. So copy all tracked + files. */ + + GitInfo gitInfo; + gitInfo.rev = "0000000000000000000000000000000000000000"; + gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + + std::set<std::string> files = + absl::StrSplit(runProgram("git", true, {"-C", uri, "ls-files", "-z"}), + absl::ByChar('\0'), absl::SkipEmpty()); + + PathFilter filter = [&](const Path& p) -> bool { + assert(absl::StartsWith(p, uri)); + std::string file(p, uri.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && absl::StartsWith(*i, prefix); + } + + return files.count(file); + }; + + gitInfo.storePath = + store->addToStore("source", uri, true, htSHA256, filter); + + return gitInfo; + } + + // clean working tree, but no ref or rev specified. Use 'HEAD'. + rev = absl::StripTrailingAsciiWhitespace( + runProgram("git", true, {"-C", uri, "rev-parse", "HEAD"})); + ref = "HEAD"s; + } + + if (!ref) { + ref = "HEAD"s; + } + + if (rev != "" && !std::regex_match(rev, revRegex)) { + throw Error("invalid Git revision '%s'", rev); + } + + deletePath(getCacheDir() + "/nix/git"); + + Path cacheDir = getCacheDir() + "/nix/gitv2/" + + hashString(htSHA256, uri).to_string(Base32, false); + + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, {"init", "--bare", cacheDir}); + } + + Path localRefFile; + if (ref->compare(0, 5, "refs/") == 0) { + localRefFile = cacheDir + "/" + *ref; + } else { + localRefFile = cacheDir + "/refs/heads/" + *ref; + } + + bool doFetch; + time_t now = time(0); + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (rev != "") { + try { + runProgram("git", true, {"-C", cacheDir, "cat-file", "-e", rev}); + doFetch = false; + } catch (ExecError& e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } + } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + static_cast<uint64_t>(st.st_mtime) + settings.tarballTtl <= + static_cast<uint64_t>(now); + } + if (doFetch) { + DLOG(INFO) << "fetching Git repository '" << uri << "'"; + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + runProgram("git", true, + {"-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, + fmt("%s:%s", *ref, *ref)}); + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } + + // FIXME: check whether rev is an ancestor of ref. + GitInfo gitInfo; + gitInfo.rev = + rev != "" ? rev + : absl::StripTrailingAsciiWhitespace(readFile(localRefFile)); + gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + + VLOG(2) << "using revision " << gitInfo.rev << " of repo '" << uri << "'"; + + std::string storeLinkName = + hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev) + .to_string(Base32, false); + Path storeLink = cacheDir + "/" + storeLinkName + ".link"; + PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", + storeLink)); // FIXME: broken + + try { + auto json = nlohmann::json::parse(readFile(storeLink)); + + assert(json["name"] == name && json["rev"] == gitInfo.rev); + + gitInfo.storePath = json["storePath"]; + + if (store->isValidPath(gitInfo.storePath)) { + gitInfo.revCount = json["revCount"]; + return gitInfo; + } + + } catch (SysError& e) { + if (e.errNo != ENOENT) { + throw; + } + } + + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto tar = runProgram("git", true, {"-C", cacheDir, "archive", gitInfo.rev}); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("tar", true, {"x", "-C", tmpDir}, tar); + + gitInfo.storePath = store->addToStore(name, tmpDir); + + gitInfo.revCount = std::stoull(runProgram( + "git", true, {"-C", cacheDir, "rev-list", "--count", gitInfo.rev})); + + nlohmann::json json; + json["storePath"] = gitInfo.storePath; + json["uri"] = uri; + json["name"] = name; + json["rev"] = gitInfo.rev; + json["revCount"] = gitInfo.revCount; + + writeFile(storeLink, json.dump()); + + return gitInfo; +} + +static void prim_fetchGit(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string url; + std::optional<std::string> ref; + std::string rev; + std::string name = "source"; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); + + for (auto& attr_iter : *args[0]->attrs) { + auto& attr = attr_iter.second; + std::string n(attr.name); + if (n == "url") { + url = + state.coerceToString(*attr.pos, *attr.value, context, false, false); + } else if (n == "ref") { + ref = state.forceStringNoCtx(*attr.value, *attr.pos); + } else if (n == "rev") { + rev = state.forceStringNoCtx(*attr.value, *attr.pos); + } else if (n == "name") { + name = state.forceStringNoCtx(*attr.value, *attr.pos); + } else { + throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", + attr.name, *attr.pos); + } + } + + if (url.empty()) { + throw EvalError(format("'url' argument required, at %1%") % pos); + } + + } else { + url = state.coerceToString(pos, *args[0], context, false, false); + } + + // FIXME: git externals probably can be used to bypass the URI + // whitelist. Ah well. + state.checkURI(url); + + auto gitInfo = exportGit(state.store, url, ref, rev, name); + + state.mkAttrs(v, 8); + mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, + PathSet({gitInfo.storePath})); + mkString(*state.allocAttr(v, state.symbols.Create("rev")), gitInfo.rev); + mkString(*state.allocAttr(v, state.symbols.Create("shortRev")), + gitInfo.shortRev); + mkInt(*state.allocAttr(v, state.symbols.Create("revCount")), + gitInfo.revCount); + + if (state.allowedPaths) { + state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); + } +} + +static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops/fetchMercurial.cc b/third_party/nix/src/libexpr/primops/fetchMercurial.cc new file mode 100644 index 0000000000..13dc61766f --- /dev/null +++ b/third_party/nix/src/libexpr/primops/fetchMercurial.cc @@ -0,0 +1,246 @@ +#include <nlohmann/json.hpp> +#include <regex> + +#include <absl/strings/ascii.h> +#include <absl/strings/match.h> +#include <absl/strings/str_split.h> +#include <glog/logging.h> +#include <sys/time.h> + +#include "libexpr/eval-inline.hh" +#include "libexpr/primops.hh" +#include "libstore/download.hh" +#include "libstore/pathlocks.hh" +#include "libstore/store-api.hh" + +using namespace std::string_literals; + +namespace nix { + +struct HgInfo { + Path storePath; + std::string branch; + std::string rev; + uint64_t revCount = 0; +}; + +std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); + +HgInfo exportMercurial(ref<Store> store, const std::string& uri, + std::string rev, const std::string& name) { + if (evalSettings.pureEval && rev == "") { + throw Error( + "in pure evaluation mode, 'fetchMercurial' requires a Mercurial " + "revision"); + } + + if (rev == "" && absl::StartsWith(uri, "/") && pathExists(uri + "/.hg")) { + bool clean = runProgram("hg", true, + {"status", "-R", uri, "--modified", "--added", + "--removed"}) == ""; + + if (!clean) { + /* This is an unclean working tree. So copy all tracked + files. */ + + DLOG(INFO) << "copying unclean Mercurial working tree '" << uri << "'"; + + HgInfo hgInfo; + hgInfo.rev = "0000000000000000000000000000000000000000"; + hgInfo.branch = absl::StripTrailingAsciiWhitespace( + runProgram("hg", true, {"branch", "-R", uri})); + + std::set<std::string> files = absl::StrSplit( + runProgram("hg", true, + {"status", "-R", uri, "--clean", "--modified", "--added", + "--no-status", "--print0"}), + absl::ByChar('\0'), absl::SkipEmpty()); + + PathFilter filter = [&](const Path& p) -> bool { + assert(absl::StartsWith(p, uri)); + std::string file(p, uri.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && absl::StartsWith(*i, prefix); + } + + return files.count(file); + }; + + hgInfo.storePath = + store->addToStore("source", uri, true, htSHA256, filter); + + return hgInfo; + } + } + + if (rev == "") { + rev = "default"; + } + + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), + hashString(htSHA256, uri).to_string(Base32, false)); + + Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, + hashString(htSHA512, rev).to_string(Base32, false)); + + /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds, + do so now. */ + time_t now = time(0); + struct stat st; + if (stat(stampFile.c_str(), &st) != 0 || + static_cast<uint64_t>(st.st_mtime) + settings.tarballTtl <= + static_cast<uint64_t>(now)) { + /* Except that if this is a commit hash that we already have, + we don't have to pull again. */ + if (!(std::regex_match(rev, commitHashRegex) && pathExists(cacheDir) && + runProgram(RunOptions("hg", {"log", "-R", cacheDir, "-r", rev, + "--template", "1"}) + .killStderr(true)) + .second == "1")) { + DLOG(INFO) << "fetching Mercurial repository '" << uri << "'"; + + if (pathExists(cacheDir)) { + try { + runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri}); + } catch (ExecError& e) { + std::string transJournal = cacheDir + "/.hg/store/journal"; + /* hg throws "abandoned transaction" error only if this file exists */ + if (pathExists(transJournal)) { + runProgram("hg", true, {"recover", "-R", cacheDir}); + runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri}); + } else { + throw ExecError(e.status, + fmt("'hg pull' %s", statusToString(e.status))); + } + } + } else { + createDirs(dirOf(cacheDir)); + runProgram("hg", true, {"clone", "--noupdate", "--", uri, cacheDir}); + } + } + + writeFile(stampFile, ""); + } + + std::vector<std::string> tokens = + absl::StrSplit(runProgram("hg", true, + {"log", "-R", cacheDir, "-r", rev, "--template", + "{node} {rev} {branch}"}), + absl::ByAnyChar(" \t\n\r"), absl::SkipEmpty()); + assert(tokens.size() == 3); + + HgInfo hgInfo; + hgInfo.rev = tokens[0]; + hgInfo.revCount = std::stoull(tokens[1]); + hgInfo.branch = tokens[2]; + + std::string storeLinkName = + hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev) + .to_string(Base32, false); + Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); + + try { + auto json = nlohmann::json::parse(readFile(storeLink)); + + assert(json["name"] == name && json["rev"] == hgInfo.rev); + + hgInfo.storePath = json["storePath"]; + + if (store->isValidPath(hgInfo.storePath)) { + DLOG(INFO) << "using cached Mercurial store path '" << hgInfo.storePath + << "'"; + return hgInfo; + } + + } catch (SysError& e) { + if (e.errNo != ENOENT) { + throw; + } + } + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("hg", true, {"archive", "-R", cacheDir, "-r", rev, tmpDir}); + + deletePath(tmpDir + "/.hg_archival.txt"); + + hgInfo.storePath = store->addToStore(name, tmpDir); + + nlohmann::json json; + json["storePath"] = hgInfo.storePath; + json["uri"] = uri; + json["name"] = name; + json["branch"] = hgInfo.branch; + json["rev"] = hgInfo.rev; + json["revCount"] = hgInfo.revCount; + + writeFile(storeLink, json.dump()); + + return hgInfo; +} + +static void prim_fetchMercurial(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string url; + std::string rev; + std::string name = "source"; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); + + for (auto& attr_iter : *args[0]->attrs) { + auto& attr = attr_iter.second; + std::string n(attr.name); + if (n == "url") { + url = + state.coerceToString(*attr.pos, *attr.value, context, false, false); + } else if (n == "rev") { + rev = state.forceStringNoCtx(*attr.value, *attr.pos); + } else if (n == "name") { + name = state.forceStringNoCtx(*attr.value, *attr.pos); + } else { + throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", + attr.name, *attr.pos); + } + } + + if (url.empty()) { + throw EvalError(format("'url' argument required, at %1%") % pos); + } + + } else { + url = state.coerceToString(pos, *args[0], context, false, false); + } + + // FIXME: git externals probably can be used to bypass the URI + // whitelist. Ah well. + state.checkURI(url); + + auto hgInfo = exportMercurial(state.store, url, rev, name); + + state.mkAttrs(v, 8); + mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, + PathSet({hgInfo.storePath})); + mkString(*state.allocAttr(v, state.symbols.Create("branch")), hgInfo.branch); + mkString(*state.allocAttr(v, state.symbols.Create("rev")), hgInfo.rev); + mkString(*state.allocAttr(v, state.symbols.Create("shortRev")), + std::string(hgInfo.rev, 0, 12)); + mkInt(*state.allocAttr(v, state.symbols.Create("revCount")), hgInfo.revCount); + + if (state.allowedPaths) { + state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); + } +} + +static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops/fromTOML.cc b/third_party/nix/src/libexpr/primops/fromTOML.cc new file mode 100644 index 0000000000..e3d2a49407 --- /dev/null +++ b/third_party/nix/src/libexpr/primops/fromTOML.cc @@ -0,0 +1,94 @@ +#include "cpptoml/cpptoml.h" +#include "libexpr/eval-inline.hh" +#include "libexpr/primops.hh" + +namespace nix { + +static void prim_fromTOML(EvalState& state, const Pos& pos, Value** args, + Value& v) { + using namespace cpptoml; + + auto toml = state.forceStringNoCtx(*args[0], pos); + + std::istringstream tomlStream(toml); + + std::function<void(Value&, std::shared_ptr<base>)> visit; + + visit = [&](Value& v, std::shared_ptr<base> t) { + if (auto t2 = t->as_table()) { + size_t size = 0; + for (auto& i : *t2) { + (void)i; + size++; + } + + state.mkAttrs(v, size); + + for (auto& i : *t2) { + auto& v2 = *state.allocAttr(v, state.symbols.Create(i.first)); + + if (auto i2 = i.second->as_table_array()) { + size_t size2 = i2->get().size(); + state.mkList(v2, size2); + for (size_t j = 0; j < size2; ++j) { + visit(*((*v2.list)[j] = state.allocValue()), i2->get()[j]); + } + } else { + visit(v2, i.second); + } + } + } + + else if (auto t2 = t->as_array()) { + size_t size = t2->get().size(); + + state.mkList(v, size); + + for (size_t i = 0; i < size; ++i) { + visit(*((*v.list)[i] = state.allocValue()), t2->get()[i]); + } + } + + // Handle cases like 'a = [[{ a = true }]]', which IMHO should be + // parsed as a array containing an array containing a table, + // but instead are parsed as an array containing a table array + // containing a table. + else if (auto t2 = t->as_table_array()) { + size_t size = t2->get().size(); + + state.mkList(v, size); + + for (size_t j = 0; j < size; ++j) { + visit(*((*v.list)[j] = state.allocValue()), t2->get()[j]); + } + } + + else if (t->is_value()) { + if (auto val = t->as<int64_t>()) { + mkInt(v, val->get()); + } else if (auto val = t->as<NixFloat>()) { + mkFloat(v, val->get()); + } else if (auto val = t->as<bool>()) { + mkBool(v, val->get()); + } else if (auto val = t->as<std::string>()) { + mkString(v, val->get()); + } else { + throw EvalError("unsupported value type in TOML"); + } + } + + else { + abort(); + } + }; + + try { + visit(v, parser(tomlStream).parse()); + } catch (std::runtime_error& e) { + throw EvalError("while parsing a TOML string at %s: %s", pos, e.what()); + } +} + +static RegisterPrimOp r("fromTOML", 1, prim_fromTOML); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/symbol-table.cc b/third_party/nix/src/libexpr/symbol-table.cc new file mode 100644 index 0000000000..2b27ca54c2 --- /dev/null +++ b/third_party/nix/src/libexpr/symbol-table.cc @@ -0,0 +1,24 @@ +#include "libexpr/symbol-table.hh" + +#include <absl/container/node_hash_set.h> +#include <absl/strings/string_view.h> + +namespace nix { + +Symbol SymbolTable::Create(absl::string_view sym) { + auto it = symbols_.emplace(sym); + const std::string* ptr = &(*it.first); + return Symbol(ptr); +} + +size_t SymbolTable::Size() const { return symbols_.size(); } + +size_t SymbolTable::TotalSize() const { + size_t n = 0; + for (auto& i : symbols_) { + n += i.size(); + } + return n; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/symbol-table.hh b/third_party/nix/src/libexpr/symbol-table.hh new file mode 100644 index 0000000000..c259965885 --- /dev/null +++ b/third_party/nix/src/libexpr/symbol-table.hh @@ -0,0 +1,69 @@ +#pragma once + +#include <absl/container/node_hash_set.h> +#include <absl/strings/string_view.h> + +namespace nix { // TODO(tazjin): ::expr + +// TODO(tazjin): Replace with a simpler struct, or get rid of. +class Symbol { + private: + const std::string* s; // pointer into SymbolTable + Symbol(const std::string* s) : s(s){}; + friend class SymbolTable; + + public: + bool operator==(const Symbol& s2) const { return s == s2.s; } + + bool operator!=(const Symbol& s2) const { return s != s2.s; } + + bool operator<(const Symbol& s2) const { return *s < *s2.s; } + + operator const std::string&() const { return *s; } + + bool set() const { return s; } + + bool empty() const { return s->empty(); } + + friend std::ostream& operator<<(std::ostream& str, const Symbol& sym); + + template <typename H> + friend H AbslHashValue(H h, const Symbol& c) { + return H::combine(std::move(h), c.s); + } +}; + +// SymbolTable is a hash-set based symbol-interning mechanism. +// +// TODO(tazjin): Figure out which things use this. AttrSets, ...? +// Is it possible this only exists because AttrSet wasn't a map? +// +// Original comment: +// +// Symbol table used by the parser and evaluator to represent and look +// up identifiers and attributes efficiently. SymbolTable::create() +// converts a string into a symbol. Symbols have the property that +// they can be compared efficiently (using a pointer equality test), +// because the symbol table stores only one copy of each string. +class SymbolTable { + public: + // Create a new symbol in this table by emplacing the provided + // string into it. + // + // The symbol will reference an existing symbol if the symbol is + // already interned. + Symbol Create(absl::string_view sym); + + // Return the number of symbols interned. + size_t Size() const; + + // Return the total size (in bytes) + size_t TotalSize() const; + + private: + // flat_hash_set does not retain pointer stability on rehashing, + // hence "interned" strings/symbols are stored on the heap. + absl::node_hash_set<std::string> symbols_; +}; + +} // namespace nix diff --git a/third_party/nix/src/libexpr/value-to-json.cc b/third_party/nix/src/libexpr/value-to-json.cc new file mode 100644 index 0000000000..a338d4eed7 --- /dev/null +++ b/third_party/nix/src/libexpr/value-to-json.cc @@ -0,0 +1,91 @@ +#include "libexpr/value-to-json.hh" + +#include <cstdlib> +#include <iomanip> + +#include "libexpr/eval-inline.hh" +#include "libutil/json.hh" +#include "libutil/util.hh" + +namespace nix { + +void printValueAsJSON(EvalState& state, bool strict, Value& v, + JSONPlaceholder& out, PathSet& context) { + checkInterrupt(); + + if (strict) { + state.forceValue(v); + } + + switch (v.type) { + case tInt: + out.write(v.integer); + break; + + case tBool: + out.write(v.boolean); + break; + + case tString: + copyContext(v, context); + out.write(v.string.s); + break; + + case tPath: + out.write(state.copyPathToStore(context, v.path)); + break; + + case tNull: + out.write(nullptr); + break; + + case tAttrs: { + auto maybeString = + state.tryAttrsToString(noPos, v, context, false, false); + if (maybeString) { + out.write(*maybeString); + break; + } + auto i = v.attrs->find(state.sOutPath); + if (i == v.attrs->end()) { + auto obj(out.object()); + StringSet names; + for (auto& j : *v.attrs) { + names.insert(j.second.name); + } + for (auto& j : names) { + auto [_, a] = *v.attrs->find(state.symbols.Create(j)); + auto placeholder(obj.placeholder(j)); + printValueAsJSON(state, strict, *a.value, placeholder, context); + } + } else { + printValueAsJSON(state, strict, *i->second.value, out, context); + } + break; + } + + case tList: { + auto list(out.list()); + for (unsigned int n = 0; n < v.listSize(); ++n) { + auto placeholder(list.placeholder()); + printValueAsJSON(state, strict, *(*v.list)[n], placeholder, context); + } + break; + } + + case tFloat: + out.write(v.fpoint); + break; + + default: + throw TypeError(format("cannot convert %1% to JSON") % showType(v)); + } +} + +void printValueAsJSON(EvalState& state, bool strict, Value& v, + std::ostream& str, PathSet& context) { + JSONPlaceholder out(str); + printValueAsJSON(state, strict, v, out, context); +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/value-to-json.hh b/third_party/nix/src/libexpr/value-to-json.hh new file mode 100644 index 0000000000..294d776045 --- /dev/null +++ b/third_party/nix/src/libexpr/value-to-json.hh @@ -0,0 +1,19 @@ +#pragma once + +#include <map> +#include <string> + +#include "libexpr/eval.hh" +#include "libexpr/nixexpr.hh" + +namespace nix { + +class JSONPlaceholder; + +void printValueAsJSON(EvalState& state, bool strict, Value& v, + JSONPlaceholder& out, PathSet& context); + +void printValueAsJSON(EvalState& state, bool strict, Value& v, + std::ostream& str, PathSet& context); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/value-to-xml.cc b/third_party/nix/src/libexpr/value-to-xml.cc new file mode 100644 index 0000000000..921973881f --- /dev/null +++ b/third_party/nix/src/libexpr/value-to-xml.cc @@ -0,0 +1,184 @@ +#include "libexpr/value-to-xml.hh" + +#include <cstdlib> + +#include "libexpr/eval-inline.hh" +#include "libutil/util.hh" +#include "libutil/xml-writer.hh" + +namespace nix { + +static XMLAttrs singletonAttrs(const std::string& name, + const std::string& value) { + XMLAttrs attrs; + attrs[name] = value; + return attrs; +} + +static void printValueAsXML(EvalState& state, bool strict, bool location, + Value& v, XMLWriter& doc, PathSet& context, + PathSet& drvsSeen); + +static void posToXML(XMLAttrs& xmlAttrs, const Pos& pos) { + xmlAttrs["path"] = pos.file.value(); + xmlAttrs["line"] = (format("%1%") % pos.line).str(); + xmlAttrs["column"] = (format("%1%") % pos.column).str(); +} + +static void showAttrs(EvalState& state, bool strict, bool location, + Bindings& attrs, XMLWriter& doc, PathSet& context, + PathSet& drvsSeen) { + StringSet names; + + for (auto& i : attrs) { + names.insert(i.second.name); + } + + for (auto& i : names) { + auto& [_, a] = *attrs.find(state.symbols.Create(i)); + + XMLAttrs xmlAttrs; + xmlAttrs["name"] = i; + if (location && a.pos != &noPos) { + posToXML(xmlAttrs, *a.pos); + } + + XMLOpenElement elem(doc, "attr", xmlAttrs); + printValueAsXML(state, strict, location, *a.value, doc, context, drvsSeen); + } +} + +static void printValueAsXML(EvalState& state, bool strict, bool location, + Value& v, XMLWriter& doc, PathSet& context, + PathSet& drvsSeen) { + checkInterrupt(); + + if (strict) { + state.forceValue(v); + } + + switch (v.type) { + case tInt: + doc.writeEmptyElement( + "int", singletonAttrs("value", (format("%1%") % v.integer).str())); + break; + + case tBool: + doc.writeEmptyElement( + "bool", singletonAttrs("value", v.boolean ? "true" : "false")); + break; + + case tString: + /* !!! show the context? */ + copyContext(v, context); + doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); + break; + + case tPath: + doc.writeEmptyElement("path", singletonAttrs("value", v.path)); + break; + + case tNull: + doc.writeEmptyElement("null"); + break; + + case tAttrs: + if (state.isDerivation(v)) { + XMLAttrs xmlAttrs; + + Bindings::iterator a = + v.attrs->find(state.symbols.Create("derivation")); + + Path drvPath; + a = v.attrs->find(state.sDrvPath); + if (a != v.attrs->end()) { + if (strict) { + state.forceValue(*a->second.value); + } + if (a->second.value->type == tString) { + xmlAttrs["drvPath"] = drvPath = a->second.value->string.s; + } + } + + a = v.attrs->find(state.sOutPath); + if (a != v.attrs->end()) { + if (strict) { + state.forceValue(*a->second.value); + } + if (a->second.value->type == tString) { + xmlAttrs["outPath"] = a->second.value->string.s; + } + } + + XMLOpenElement _(doc, "derivation", xmlAttrs); + + if (!drvPath.empty() && drvsSeen.find(drvPath) == drvsSeen.end()) { + drvsSeen.insert(drvPath); + showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); + } else { + doc.writeEmptyElement("repeated"); + } + } + + else { + XMLOpenElement _(doc, "attrs"); + showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); + } + + break; + + case tList: { + XMLOpenElement _(doc, "list"); + for (unsigned int n = 0; n < v.listSize(); ++n) { + printValueAsXML(state, strict, location, *(*v.list)[n], doc, context, + drvsSeen); + } + break; + } + + case tLambda: { + XMLAttrs xmlAttrs; + if (location) { + posToXML(xmlAttrs, v.lambda.fun->pos); + } + XMLOpenElement _(doc, "function", xmlAttrs); + + if (v.lambda.fun->matchAttrs) { + XMLAttrs attrs; + if (!v.lambda.fun->arg.empty()) { + attrs["name"] = v.lambda.fun->arg; + } + if (v.lambda.fun->formals->ellipsis) { + attrs["ellipsis"] = "1"; + } + XMLOpenElement _(doc, "attrspat", attrs); + for (auto& i : v.lambda.fun->formals->formals) { + doc.writeEmptyElement("attr", singletonAttrs("name", i.name)); + } + } else { + doc.writeEmptyElement("varpat", + singletonAttrs("name", v.lambda.fun->arg)); + } + + break; + } + + case tFloat: + doc.writeEmptyElement( + "float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); + break; + + default: + doc.writeEmptyElement("unevaluated"); + } +} + +void printValueAsXML(EvalState& state, bool strict, bool location, Value& v, + std::ostream& out, PathSet& context) { + XMLWriter doc(true, out); + XMLOpenElement root(doc, "expr"); + PathSet drvsSeen; + printValueAsXML(state, strict, location, v, doc, context, drvsSeen); +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/value-to-xml.hh b/third_party/nix/src/libexpr/value-to-xml.hh new file mode 100644 index 0000000000..18c5279236 --- /dev/null +++ b/third_party/nix/src/libexpr/value-to-xml.hh @@ -0,0 +1,14 @@ +#pragma once + +#include <map> +#include <string> + +#include "libexpr/eval.hh" +#include "libexpr/nixexpr.hh" + +namespace nix { + +void printValueAsXML(EvalState& state, bool strict, bool location, Value& v, + std::ostream& out, PathSet& context); + +} diff --git a/third_party/nix/src/libexpr/value.cc b/third_party/nix/src/libexpr/value.cc new file mode 100644 index 0000000000..93fe187478 --- /dev/null +++ b/third_party/nix/src/libexpr/value.cc @@ -0,0 +1,121 @@ +#include "libexpr/value.hh" + +#include <glog/logging.h> + +namespace nix { + +Value::Value(const Value& copy) { *this = copy; } + +Value::Value(Value&& move) { *this = move; } + +Value& Value::operator=(const Value& copy) { + if (type != copy.type) { + memset(this, 0, sizeof(*this)); + } + type = copy.type; + switch (type) { + case tInt: + integer = copy.integer; + break; + case tBool: + boolean = copy.boolean; + break; + case tString: + string = copy.string; + break; + case tPath: + path = copy.path; + break; + case tNull: + /* no fields */ + break; + case tAttrs: + attrs = copy.attrs; + break; + case tList: + list = copy.list; + break; + case tThunk: + thunk = copy.thunk; + break; + case tApp: + app = copy.app; + break; + case tLambda: + lambda = copy.lambda; + break; + case tBlackhole: + /* no fields */ + break; + case tPrimOp: + primOp = copy.primOp; + break; + case tPrimOpApp: + primOpApp = copy.primOpApp; + break; + case _reserved1: + LOG(FATAL) << "attempted to assign a tExternal value"; + break; + case tFloat: + fpoint = copy.fpoint; + break; + } + return *this; +} + +Value& Value::operator=(Value&& move) { + if (type != move.type) { + memset(this, 0, sizeof(*this)); + } + type = move.type; + switch (type) { + case tInt: + integer = move.integer; + break; + case tBool: + boolean = move.boolean; + break; + case tString: + string = move.string; + break; + case tPath: + path = move.path; + break; + case tNull: + /* no fields */ + break; + case tAttrs: + attrs = move.attrs; + break; + case tList: + list = move.list; + break; + case tThunk: + thunk = move.thunk; + break; + case tApp: + app = move.app; + break; + case tLambda: + lambda = move.lambda; + break; + case tBlackhole: + /* no fields */ + break; + case tPrimOp: + primOp = move.primOp; + break; + case tPrimOpApp: + primOpApp = move.primOpApp; + break; + case _reserved1: + LOG(FATAL) << "attempted to assign a tExternal value"; + break; + case tFloat: + fpoint = move.fpoint; + break; + } + return *this; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/value.hh b/third_party/nix/src/libexpr/value.hh new file mode 100644 index 0000000000..82021c77c4 --- /dev/null +++ b/third_party/nix/src/libexpr/value.hh @@ -0,0 +1,191 @@ +#pragma once + +#include <tuple> +#include <vector> + +#include "libexpr/symbol-table.hh" +#include "libutil/types.hh" + +namespace nix { + +using ValueType = enum { + tInt = 1, + tBool, + tString, + tPath, + tNull, + tAttrs, + tList, + tThunk, + tApp, + tLambda, + tBlackhole, + tPrimOp, + tPrimOpApp, + _reserved1, // formerly tExternal + tFloat +}; + +class Bindings; +struct Env; +struct Expr; +struct ExprLambda; +struct PrimOp; +struct PrimOp; +class Symbol; + +typedef int64_t NixInt; +typedef double NixFloat; + +// Forward declaration of Value is required because the following +// types are mutually recursive. +// +// TODO(tazjin): Really, these types need some serious refactoring. +struct Value; + +/* Strings in the evaluator carry a so-called `context' which + is a list of strings representing store paths. This is to + allow users to write things like + + "--with-freetype2-library=" + freetype + "/lib" + + where `freetype' is a derivation (or a source to be copied + to the store). If we just concatenated the strings without + keeping track of the referenced store paths, then if the + string is used as a derivation attribute, the derivation + will not have the correct dependencies in its inputDrvs and + inputSrcs. + + The semantics of the context is as follows: when a string + with context C is used as a derivation attribute, then the + derivations in C will be added to the inputDrvs of the + derivation, and the other store paths in C will be added to + the inputSrcs of the derivations. + + For canonicity, the store paths should be in sorted order. */ +struct NixString { + const char* s; + const char** context; // must be in sorted order +}; + +struct NixThunk { + Env* env; + Expr* expr; +}; + +struct NixApp { + Value *left, *right; +}; + +struct NixLambda { + Env* env; + ExprLambda* fun; +}; + +struct NixPrimOpApp { + Value *left, *right; +}; + +using NixList = std::vector<Value*>; + +struct Value { + ValueType type; + union { // TODO(tazjin): std::variant + NixInt integer; + bool boolean; + NixString string; + const char* path; + std::shared_ptr<Bindings> attrs; + std::shared_ptr<NixList> list; + NixThunk thunk; + NixApp app; // TODO(tazjin): "app"? + NixLambda lambda; + std::shared_ptr<PrimOp> primOp; + NixPrimOpApp primOpApp; + NixFloat fpoint; + }; + + Value() : type(tInt), attrs(nullptr) { + static_assert(offsetof(Value, attrs) + sizeof(attrs) == sizeof(Value)); + } + + Value(const Value& copy); + Value(Value&& move); + ~Value() {} + Value& operator=(const Value& copy); + Value& operator=(Value&& move); + + bool isList() const { return type == tList; } + + size_t listSize() const { return list->size(); } +}; + +/* After overwriting an app node, be sure to clear pointers in the + Value to ensure that the target isn't kept alive unnecessarily. */ +static inline void clearValue(Value& v) { v.app.left = v.app.right = 0; } + +static inline void mkInt(Value& v, NixInt n) { + clearValue(v); + v.type = tInt; + v.integer = n; +} + +static inline void mkFloat(Value& v, NixFloat n) { + clearValue(v); + v.type = tFloat; + v.fpoint = n; +} + +static inline void mkBool(Value& v, bool b) { + clearValue(v); + v.type = tBool; + v.boolean = b; +} + +static inline void mkNull(Value& v) { + clearValue(v); + v.type = tNull; +} + +static inline void mkApp(Value& v, Value& left, Value& right) { + v.type = tApp; + v.app.left = &left; + v.app.right = &right; +} + +static inline void mkPrimOpApp(Value& v, Value& left, Value& right) { + v.type = tPrimOpApp; + v.app.left = &left; + v.app.right = &right; +} + +static inline void mkStringNoCopy(Value& v, const char* s) { + v.type = tString; + v.string.s = s; + v.string.context = 0; +} + +static inline void mkString(Value& v, const Symbol& s) { + mkStringNoCopy(v, ((const std::string&)s).c_str()); +} + +void mkString(Value& v, const char* s); + +static inline void mkPathNoCopy(Value& v, const char* s) { + clearValue(v); + v.type = tPath; + v.path = s; +} + +void mkPath(Value& v, const char* s); + +/* Compute the size in bytes of the given value, including all values + and environments reachable from it. Static expressions (Exprs) are + not included. */ +size_t valueSize(const Value& v); + +using ValueMap = std::map<Symbol, Value*>; + +std::shared_ptr<Value*> allocRootValue(Value* v); + +} // namespace nix |