diff options
Diffstat (limited to 'src')
75 files changed, 2235 insertions, 1192 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 445006b327fd..dbf8fe1b8f8a 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -64,6 +64,8 @@ int main (int argc, char * * argv) settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work + initPlugins(); + auto store = openStore().cast<LocalStore>(); /* It would be more appropriate to use $XDG_RUNTIME_DIR, since @@ -218,9 +220,11 @@ connected: signal(SIGALRM, old); } + auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; + { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); - copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs); + copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, substitute); } uploadLock = -1; @@ -239,8 +243,8 @@ connected: if (!missing.empty()) { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); - setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */ - copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs); + store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */ + copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, substitute); } return; diff --git a/src/buildenv/buildenv.cc b/src/buildenv/buildenv.cc index eddb9fdaa8d2..2afad913ac6b 100644 --- a/src/buildenv/buildenv.cc +++ b/src/buildenv/buildenv.cc @@ -74,10 +74,10 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) auto prevPriority = priorities[dstFile]; if (prevPriority == priority) throw Error(format( - "Packages '%1%' and '%2%' have the same priority '%3%'" + "packages '%1%' and '%2%' have the same priority %3%; " "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " "to change the priority of one of the conflicting packages" - " ('0' being the highest priority)" + " (0 being the highest priority)" ) % srcFile % target % priority); if (prevPriority < priority) continue; diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 910428c02686..b284daa3c2f7 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -7,13 +7,14 @@ namespace nix { +/* Note: Various places expect the allocated memory to be zeroed. */ static void * allocBytes(size_t n) { void * p; #if HAVE_BOEHMGC p = GC_malloc(n); #else - p = malloc(n); + p = calloc(n, 1); #endif if (!p) throw std::bad_alloc(); return p; diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index e1fc2bf6d796..3119a1848af2 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -83,7 +83,7 @@ public: for (size_t n = 0; n < size_; n++) res.emplace_back(&attrs[n]); std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) { - return (string) a->name < (string) b->name; + return (const string &) a->name < (const string &) b->name; }); return res; } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 63de2d60a147..f94c23ea72bb 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -43,13 +43,14 @@ static char * dupString(const char * s) } +/* Note: Various places expect the allocated memory to be zeroed. */ static void * allocBytes(size_t n) { void * p; #if HAVE_BOEHMGC p = GC_malloc(n); #else - p = malloc(n); + p = calloc(n, 1); #endif if (!p) throw std::bad_alloc(); return p; @@ -293,6 +294,10 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) , sWrong(symbols.create("wrong")) , sStructuredAttrs(symbols.create("__structuredAttrs")) , sBuilder(symbols.create("builder")) + , sArgs(symbols.create("args")) + , sOutputHash(symbols.create("outputHash")) + , sOutputHashAlgo(symbols.create("outputHashAlgo")) + , sOutputHashMode(symbols.create("outputHashMode")) , repair(NoRepair) , store(store) , baseEnv(allocEnv(128)) @@ -300,15 +305,24 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) { countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0"; - restricted = settings.restrictEval; - assert(gcInitialised); /* Initialise the Nix expression search path. */ - Strings paths = parseNixPath(getEnv("NIX_PATH", "")); - for (auto & i : _searchPath) addToSearchPath(i); - for (auto & i : paths) addToSearchPath(i); - addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs"); + if (!settings.pureEval) { + Strings paths = parseNixPath(getEnv("NIX_PATH", "")); + for (auto & i : _searchPath) addToSearchPath(i); + for (auto & i : paths) addToSearchPath(i); + } + addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); + + if (settings.restrictEval || settings.pureEval) { + allowedPaths = PathSet(); + for (auto & i : searchPath) { + auto r = resolveSearchPathElem(i); + if (!r.first) continue; + allowedPaths->insert(r.second); + } + } clearValue(vEmptySet); vEmptySet.type = tAttrs; @@ -326,38 +340,36 @@ EvalState::~EvalState() Path EvalState::checkSourcePath(const Path & path_) { - if (!restricted) return path_; + if (!allowedPaths) return path_; + + bool found = false; + + for (auto & i : *allowedPaths) { + if (isDirOrInDir(path_, i)) { + found = true; + break; + } + } + + if (!found) + throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path_); /* Resolve symlinks. */ debug(format("checking access to '%s'") % path_); Path path = canonPath(path_, true); - for (auto & i : searchPath) { - auto r = resolveSearchPathElem(i); - if (!r.first) continue; - if (path == r.second || isInDir(path, r.second)) + for (auto & i : *allowedPaths) { + if (isDirOrInDir(path, i)) return path; } - /* To support import-from-derivation, allow access to anything in - the store. FIXME: only allow access to paths that have been - constructed by this evaluation. */ - if (store->isInStore(path)) return path; - -#if 0 - /* Hack to support the chroot dependencies of corepkgs (see - corepkgs/config.nix.in). */ - if (path == settings.nixPrefix && isStorePath(settings.nixPrefix)) - return path; -#endif - - throw RestrictedPathError(format("access to path '%1%' is forbidden in restricted mode") % path_); + throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path); } void EvalState::checkURI(const std::string & uri) { - if (!restricted) return; + if (!settings.restrictEval) return; /* 'uri' should be equal to a prefix, or in a subdirectory of a prefix. Thus, the prefix https://github.co does not permit @@ -371,11 +383,33 @@ void EvalState::checkURI(const std::string & uri) && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) return; + /* If the URI is a path, then check it against allowedPaths as + well. */ + if (hasPrefix(uri, "/")) { + checkSourcePath(uri); + return; + } + + if (hasPrefix(uri, "file://")) { + checkSourcePath(std::string(uri, 7)); + return; + } + throw RestrictedPathError("access to URI '%s' is forbidden in restricted mode", uri); } -void EvalState::addConstant(const string & name, Value & v) +Path EvalState::toRealPath(const Path & path, const PathSet & context) +{ + // FIXME: check whether 'path' is in 'context'. + return + !context.empty() && store->isInStore(path) + ? store->toRealPath(path) + : path; +}; + + +Value * EvalState::addConstant(const string & name, Value & v) { Value * v2 = allocValue(); *v2 = v; @@ -383,12 +417,18 @@ void EvalState::addConstant(const string & name, Value & v) baseEnv.values[baseEnvDispl++] = v2; string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); + return v2; } -void EvalState::addPrimOp(const string & name, +Value * EvalState::addPrimOp(const string & name, unsigned int arity, PrimOpFun primOp) { + if (arity == 0) { + Value v; + primOp(*this, noPos, nullptr, v); + return addConstant(name, v); + } Value * v = allocValue(); string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; Symbol sym = symbols.create(name2); @@ -397,6 +437,7 @@ void EvalState::addPrimOp(const string & name, staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(sym, v)); + return v; } @@ -546,9 +587,7 @@ Env & EvalState::allocEnv(unsigned int size) Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); env->size = size; - /* Clear the values because maybeThunk() and lookupVar fromWith expect this. */ - for (unsigned i = 0; i < size; ++i) - env->values[i] = 0; + /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ return *env; } @@ -649,8 +688,10 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) } -void EvalState::evalFile(const Path & path, Value & v) +void EvalState::evalFile(const Path & path_, Value & v) { + auto path = checkSourcePath(path_); + FileEvalCache::iterator i; if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) { v = i->second; @@ -1546,7 +1587,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) dstPath = srcToStore[path]; else { dstPath = settings.readOnlyMode - ? store->computeStorePathForPath(checkSourcePath(path)).first + ? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); srcToStore[path] = dstPath; printMsg(lvlChatty, format("copied source '%1%' -> '%2%'") @@ -1668,10 +1709,13 @@ void EvalState::printStats() printMsg(v, format(" time elapsed: %1%") % cpuTime); printMsg(v, format(" size of a value: %1%") % sizeof(Value)); printMsg(v, format(" size of an attr: %1%") % sizeof(Attr)); - printMsg(v, format(" environments allocated: %1% (%2% bytes)") % nrEnvs % bEnvs); - printMsg(v, format(" list elements: %1% (%2% bytes)") % nrListElems % bLists); + printMsg(v, format(" environments allocated count: %1%") % nrEnvs); + printMsg(v, format(" environments allocated bytes: %1%") % bEnvs); + printMsg(v, format(" list elements count: %1%") % nrListElems); + printMsg(v, format(" list elements bytes: %1%") % bLists); printMsg(v, format(" list concatenations: %1%") % nrListConcats); - printMsg(v, format(" values allocated: %1% (%2% bytes)") % nrValues % bValues); + printMsg(v, format(" values allocated count: %1%") % nrValues); + printMsg(v, format(" values allocated bytes: %1%") % bValues); printMsg(v, format(" sets allocated: %1% (%2% bytes)") % nrAttrsets % bAttrsets); printMsg(v, format(" right-biased unions: %1%") % nrOpUpdates); printMsg(v, format(" values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f0ab1435bff3..9d8799b7906b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -69,16 +69,17 @@ public: const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, - sRight, sWrong, sStructuredAttrs, sBuilder; + sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sOutputHash, sOutputHashAlgo, sOutputHashMode; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they already exist there. */ RepairFlag repair; - /* If set, don't allow access to files outside of the Nix search - path or to environment variables. */ - bool restricted; + /* The allowed filesystem paths in restricted or pure evaluation + mode. */ + std::experimental::optional<PathSet> allowedPaths; Value vEmptySet; @@ -112,6 +113,15 @@ public: void checkURI(const std::string & uri); + /* When using a diverted store and 'path' is in the Nix store, map + 'path' to the diverted location (e.g. /nix/store/foo is mapped + to /home/alice/my-nix/nix/store/foo). However, this is only + done if the context is not empty, since otherwise we're + probably trying to read from the actual /nix/store. This is + intended to distinguish between import-from-derivation and + sources stored in the actual /nix/store. */ + Path toRealPath(const Path & path, const PathSet & context); + /* Parse a Nix expression from the specified file. */ Expr * parseExprFromFile(const Path & path); Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv); @@ -201,9 +211,9 @@ private: void createBaseEnv(); - void addConstant(const string & name, Value & v); + Value * addConstant(const string & name, Value & v); - void addPrimOp(const string & name, + Value * addPrimOp(const string & name, unsigned int arity, PrimOpFun primOp); public: diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 9380de3a66b3..8b1404595548 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -106,10 +106,16 @@ static void parseJSON(EvalState & state, const char * & s, Value & v) tmp_number += *s++; } - if (number_type == tFloat) - mkFloat(v, stod(tmp_number)); - else - mkInt(v, stoi(tmp_number)); + try { + if (number_type == tFloat) + mkFloat(v, stod(tmp_number)); + else + mkInt(v, stoi(tmp_number)); + } catch (std::invalid_argument e) { + throw JSONParseError("invalid JSON number"); + } catch (std::out_of_range e) { + throw JSONParseError("out-of-range JSON number"); + } } else if (strncmp(s, "true", 4) == 0) { diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 28a0a6a87896..e5e01fb5831a 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -49,9 +49,10 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) } -static Expr * unescapeStr(SymbolTable & symbols, const char * s) +static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length) { string t; + t.reserve(length); char c; while ((c = *s++)) { if (c == '\\') { @@ -150,7 +151,7 @@ or { return OR_KW; } /* It is impossible to match strings ending with '$' with one regex because trailing contexts are only valid at the end of a rule. (A sane but undocumented limitation.) */ - yylval->e = unescapeStr(data->symbols, yytext); + yylval->e = unescapeStr(data->symbols, yytext, yyleng); return STR; } <STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } @@ -178,7 +179,7 @@ or { return OR_KW; } return IND_STR; } <IND_STRING>\'\'\\. { - yylval->e = unescapeStr(data->symbols, yytext + 2); + yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2); return IND_STR; } <IND_STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } diff --git a/src/libexpr/names.cc b/src/libexpr/names.cc index 6d78d2116121..382088c78872 100644 --- a/src/libexpr/names.cc +++ b/src/libexpr/names.cc @@ -41,7 +41,7 @@ bool DrvName::matches(DrvName & n) } -static string nextComponent(string::const_iterator & p, +string nextComponent(string::const_iterator & p, const string::const_iterator end) { /* Skip any dots and dashes (component separators). */ diff --git a/src/libexpr/names.hh b/src/libexpr/names.hh index 9667fc96fd0f..13c3093e77b0 100644 --- a/src/libexpr/names.hh +++ b/src/libexpr/names.hh @@ -24,6 +24,8 @@ private: typedef list<DrvName> DrvNames; +string nextComponent(string::const_iterator & p, + const string::const_iterator end); int compareVersions(const string & v1, const string & v2); DrvNames drvNamesFromArgs(const Strings & opArgs); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e3b5dfb420b4..6778023f506d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -39,7 +39,7 @@ std::pair<string, string> decodeContext(const string & s) size_t index = s.find("!", 1); return std::pair<string, string>(string(s, index + 1), string(s, 1, index - 1)); } else - return std::pair<string, string>(s.at(0) == '/' ? s: string(s, 1), ""); + return std::pair<string, string>(s.at(0) == '/' ? s : string(s, 1), ""); } @@ -49,24 +49,38 @@ InvalidPathError::InvalidPathError(const Path & path) : void EvalState::realiseContext(const PathSet & context) { PathSet drvs; + for (auto & i : context) { std::pair<string, string> decoded = decodeContext(i); Path ctx = decoded.first; assert(store->isStorePath(ctx)); if (!store->isValidPath(ctx)) throw InvalidPathError(ctx); - if (!decoded.second.empty() && nix::isDerivation(ctx)) + if (!decoded.second.empty() && nix::isDerivation(ctx)) { drvs.insert(decoded.first + "!" + decoded.second); + + /* Add the output of this derivation to the allowed + paths. */ + if (allowedPaths) { + auto drv = store->derivationFromPath(decoded.first); + DerivationOutputs::iterator i = drv.outputs.find(decoded.second); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second); + allowedPaths->insert(i->second.path); + } + } } - if (!drvs.empty()) { - if (!settings.enableImportFromDerivation) - throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin())); - /* For performance, prefetch all substitute info. */ - PathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; - store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); - store->buildPaths(drvs); - } + + if (drvs.empty()) return; + + if (!settings.enableImportFromDerivation) + throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin())); + + /* For performance, prefetch all substitute info. */ + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); + store->buildPaths(drvs); } @@ -84,10 +98,10 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args % path % e.path % pos); } - path = state.checkSourcePath(path); + Path realPath = state.checkSourcePath(state.toRealPath(path, context)); if (state.store->isStorePath(path) && state.store->isValidPath(path) && isDerivation(path)) { - Derivation drv = readDerivation(path); + Derivation drv = readDerivation(realPath); Value & w = *state.allocValue(); state.mkAttrs(w, 3 + drv.outputs.size()); Value * v2 = state.allocAttr(w, state.sDrvPath); @@ -114,7 +128,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args } else { state.forceAttrs(*args[0]); if (args[0]->attrs->empty()) - state.evalFile(path, v); + state.evalFile(realPath, v); else { Env * env = &state.allocEnv(args[0]->attrs->size()); env->up = &state.baseEnv; @@ -127,8 +141,8 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args env->values[displ++] = attr.value; } - printTalkative("evaluating file '%1%'", path); - Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); + printTalkative("evaluating file '%1%'", realPath); + Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv); e->eval(state, *env, v); } @@ -439,7 +453,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v) { string name = state.forceStringNoCtx(*args[0], pos); - mkString(v, state.restricted ? "" : getEnv(name)); + mkString(v, settings.restrictEval || settings.pureEval ? "" : getEnv(name)); } @@ -539,7 +553,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : args[0]->attrs->lexicographicOrder()) { if (i->name == state.sIgnoreNulls) continue; - string key = i->name; + const string & key = i->name; vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string & s) { @@ -575,7 +589,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ - if (key == "args") { + if (i->name == state.sArgs) { state.forceList(*i->value, pos); for (unsigned int n = 0; n < i->value->listSize(); ++n) { string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); @@ -598,15 +612,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * drv.builder = state.forceString(*i->value, context, posDrvName); else if (i->name == state.sSystem) drv.platform = state.forceStringNoCtx(*i->value, posDrvName); - else if (i->name == state.sName) - drvName = state.forceStringNoCtx(*i->value, posDrvName); - else if (key == "outputHash") + else if (i->name == state.sOutputHash) outputHash = state.forceStringNoCtx(*i->value, posDrvName); - else if (key == "outputHashAlgo") + else if (i->name == state.sOutputHashAlgo) outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName); - else if (key == "outputHashMode") + else if (i->name == state.sOutputHashMode) handleHashMode(state.forceStringNoCtx(*i->value, posDrvName)); - else if (key == "outputs") { + else if (i->name == state.sOutputs) { /* Require โoutputsโ to be a list of strings. */ state.forceList(*i->value, posDrvName); Strings ss; @@ -620,14 +632,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * drv.env.emplace(key, s); if (i->name == state.sBuilder) drv.builder = s; else if (i->name == state.sSystem) drv.platform = s; - else if (i->name == state.sName) { - drvName = s; - printMsg(lvlVomit, format("derivation name is '%1%'") % drvName); - } - else if (key == "outputHash") outputHash = s; - else if (key == "outputHashAlgo") outputHashAlgo = s; - else if (key == "outputHashMode") handleHashMode(s); - else if (key == "outputs") + else if (i->name == state.sOutputHash) outputHash = s; + else if (i->name == state.sOutputHashAlgo) outputHashAlgo = s; + else if (i->name == state.sOutputHashMode) handleHashMode(s); + else if (i->name == state.sOutputs) handleOutputs(tokenizeString<Strings>(s)); } @@ -863,7 +871,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%") % path % e.path % pos); } - string s = readFile(state.checkSourcePath(path)); + string s = readFile(state.checkSourcePath(state.toRealPath(path, context))); if (s.find((char) 0) != string::npos) throw Error(format("the contents of the file '%1%' cannot be represented as a Nix string") % path); mkString(v, s.c_str()); @@ -1009,20 +1017,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu } -static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, + Value * filterFun, bool recursive, const Hash & expectedHash, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); - if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); - - state.forceValue(*args[0]); - if (args[0]->type != tLambda) - throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); - - path = state.checkSourcePath(path); - - PathFilter filter = [&](const Path & path) { + const auto path = settings.pureEval && expectedHash ? + path_ : + state.checkSourcePath(path_); + PathFilter filter = filterFun ? ([&](const Path & path) { auto st = lstat(path); /* Call the filter function. The first argument is the path, @@ -1031,7 +1032,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args mkString(arg1, path); Value fun2; - state.callFunction(*args[0], arg1, fun2, noPos); + state.callFunction(*filterFun, arg1, fun2, noPos); Value arg2; mkString(arg2, @@ -1044,16 +1045,79 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args state.callFunction(fun2, arg2, res, noPos); return state.forceBool(res, pos); - }; + }) : defaultPathFilter; - Path dstPath = settings.readOnlyMode - ? state.store->computeStorePathForPath(path, true, htSHA256, filter).first - : state.store->addToStore(baseNameOf(path), path, true, htSHA256, filter, state.repair); + Path expectedStorePath; + if (expectedHash) { + expectedStorePath = + state.store->makeFixedOutputPath(recursive, expectedHash, name); + } + Path dstPath; + if (!expectedHash || !state.store->isValidPath(expectedStorePath)) { + dstPath = settings.readOnlyMode + ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first + : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair); + if (expectedHash && expectedStorePath != dstPath) { + throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path); + } + } else + dstPath = expectedStorePath; mkString(v, dstPath, {dstPath}); } +static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + if (!context.empty()) + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); + + state.forceValue(*args[0]); + if (args[0]->type != tLambda) + throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); + + addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v); +} + +static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + Path path; + string name; + Value * filterFun = nullptr; + auto recursive = true; + Hash expectedHash; + + for (auto & attr : *args[0]->attrs) { + const string & n(attr.name); + if (n == "path") { + PathSet context; + path = state.coerceToPath(*attr.pos, *attr.value, context); + if (!context.empty()) + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos); + } else if (attr.name == state.sName) + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "filter") { + state.forceValue(*attr.value); + filterFun = attr.value; + } else if (n == "recursive") + recursive = state.forceBool(*attr.value, *attr.pos); + else if (n == "sha256") + expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else + throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos); + } + if (path.empty()) + throw EvalError(format("'path' required, at %1%") % pos); + if (name.empty()) + name = baseNameOf(path); + + addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); +} + + /************************************************************* * Sets *************************************************************/ @@ -1068,8 +1132,11 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V state.mkList(v, args[0]->attrs->size()); size_t n = 0; - for (auto & i : args[0]->attrs->lexicographicOrder()) - mkString(*(v.listElems()[n++] = state.allocValue()), i->name); + for (auto & i : *args[0]->attrs) + mkString(*(v.listElems()[n++] = state.allocValue()), i.name); + + std::sort(v.listElems(), v.listElems() + n, + [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); } @@ -1653,6 +1720,14 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, } +static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + state.forceString(*args[0], context, pos); + mkBool(v, !context.empty()); +} + + /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a builder without causing the derivation to be built (for instance, in the derivation that builds NARs in nix-push, when doing @@ -1838,21 +1913,32 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar auto s = state.forceString(*args[2], context, pos); string res; - for (size_t p = 0; p < s.size(); ) { + // Loops one past last character to handle the case where 'from' contains an empty string. + for (size_t p = 0; p <= s.size(); ) { bool found = false; auto i = from.begin(); auto j = to.begin(); for (; i != from.end(); ++i, ++j) if (s.compare(p, i->size(), *i) == 0) { found = true; - p += i->size(); res += j->first; + if (i->empty()) { + if (p < s.size()) + res += s[p]; + p++; + } else { + p += i->size(); + } for (auto& path : j->second) context.insert(path); j->second.clear(); break; } - if (!found) res += s[p++]; + if (!found) { + if (p < s.size()) + res += s[p]; + p++; + } } mkString(v, res, context); @@ -1883,6 +1969,26 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a } +static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string version = state.forceStringNoCtx(*args[0], pos); + auto iter = version.cbegin(); + Strings components; + while (iter != version.cend()) { + auto component = nextComponent(iter, version.cend()); + if (component.empty()) + break; + components.emplace_back(std::move(component)); + } + state.mkList(v, components.size()); + unsigned int n = 0; + for (auto & component : components) { + auto listElem = v.listElems()[n++] = state.allocValue(); + mkString(*listElem, std::move(component)); + } +} + + /************************************************************* * Networking *************************************************************/ @@ -1921,7 +2027,14 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, state.checkURI(url); + if (settings.pureEval && !expectedHash) + throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash); + + if (state.allowedPaths) + state.allowedPaths->insert(res); + mkString(v, res, PathSet({res})); } @@ -1973,11 +2086,24 @@ void EvalState::createBaseEnv() mkNull(v); addConstant("null", v); - mkInt(v, time(0)); - addConstant("__currentTime", v); + auto vThrow = addPrimOp("throw", 1, prim_throw); - mkString(v, settings.thisSystem); - addConstant("__currentSystem", v); + auto addPurityError = [&](const std::string & name) { + Value * v2 = allocValue(); + mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name)); + mkApp(v, *vThrow, *v2); + addConstant(name, v); + }; + + if (!settings.pureEval) { + mkInt(v, time(0)); + addConstant("__currentTime", v); + } + + if (!settings.pureEval) { + mkString(v, settings.thisSystem); + addConstant("__currentSystem", v); + } mkString(v, nixVersion); addConstant("__nixVersion", v); @@ -1993,10 +2119,10 @@ void EvalState::createBaseEnv() addConstant("__langVersion", v); // Miscellaneous - addPrimOp("scopedImport", 2, prim_scopedImport); + auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport); Value * v2 = allocValue(); mkAttrs(*v2, 0); - mkApp(v, *baseEnv.values[baseEnvDispl - 1], *v2); + mkApp(v, *vScopedImport, *v2); forceValue(v); addConstant("import", v); if (settings.enableNativeCode) { @@ -2012,7 +2138,6 @@ void EvalState::createBaseEnv() addPrimOp("__isBool", 1, prim_isBool); addPrimOp("__genericClosure", 1, prim_genericClosure); addPrimOp("abort", 1, prim_abort); - addPrimOp("throw", 1, prim_throw); addPrimOp("__addErrorContext", 2, prim_addErrorContext); addPrimOp("__tryEval", 1, prim_tryEval); addPrimOp("__getEnv", 1, prim_getEnv); @@ -2027,7 +2152,10 @@ void EvalState::createBaseEnv() // Paths addPrimOp("__toPath", 1, prim_toPath); - addPrimOp("__storePath", 1, prim_storePath); + if (settings.pureEval) + addPurityError("__storePath"); + else + addPrimOp("__storePath", 1, prim_storePath); addPrimOp("__pathExists", 1, prim_pathExists); addPrimOp("baseNameOf", 1, prim_baseNameOf); addPrimOp("dirOf", 1, prim_dirOf); @@ -2041,6 +2169,7 @@ void EvalState::createBaseEnv() addPrimOp("__fromJSON", 1, prim_fromJSON); addPrimOp("__toFile", 2, prim_toFile); addPrimOp("__filterSource", 2, prim_filterSource); + addPrimOp("__path", 1, prim_path); // Sets addPrimOp("__attrNames", 1, prim_attrNames); @@ -2083,6 +2212,7 @@ void EvalState::createBaseEnv() addPrimOp("toString", 1, prim_toString); addPrimOp("__substring", 3, prim_substring); addPrimOp("__stringLength", 1, prim_stringLength); + addPrimOp("__hasContext", 1, prim_hasContext); addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); addPrimOp("__hashString", 2, prim_hashString); @@ -2094,6 +2224,7 @@ void EvalState::createBaseEnv() // Versions addPrimOp("__parseDrvName", 1, prim_parseDrvName); addPrimOp("__compareVersions", 2, prim_compareVersions); + addPrimOp("__splitVersion", 1, prim_splitVersion); // Derivations addPrimOp("derivationStrict", 1, prim_derivationStrict); @@ -2105,7 +2236,7 @@ void EvalState::createBaseEnv() /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ - string path = settings.nixDataDir + "/nix/corepkgs/derivation.nix"; + string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); sDerivationNix = symbols.create(path); evalFile(path, v); addConstant("derivation", v); diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 39d23b04a5ce..31bf3f84f6c7 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -9,6 +9,9 @@ struct RegisterPrimOp { typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps; static PrimOps * primOps; + /* You can register a constant by passing an arity of 0. fun + will get called during EvalState initialization, so there + may be primops not yet added and builtins is not yet sorted. */ RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun); }; diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index e92e0638031f..9fc0d46626dd 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -22,10 +22,15 @@ struct GitInfo uint64_t revCount = 0; }; +std::regex revRegex("^[0-9a-fA-F]{40}$"); + GitInfo exportGit(ref<Store> store, const std::string & uri, - std::experimental::optional<std::string> ref, const std::string & rev, + std::experimental::optional<std::string> ref, std::string rev, const std::string & name) { + if (settings.pureEval && rev == "") + throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); + if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { bool clean = true; @@ -68,20 +73,20 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, return gitInfo; } + + // clean working tree, but no ref or rev specified. Use 'HEAD'. + rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" })); + ref = "HEAD"s; } - if (!ref) ref = "master"s; + if (!ref) ref = "HEAD"s; - if (rev != "") { - std::regex revRegex("^[0-9a-fA-F]{40}$"); - if (!std::regex_match(rev, revRegex)) - throw Error("invalid Git revision '%s'", rev); - } + if (rev != "" && !std::regex_match(rev, revRegex)) + throw Error("invalid Git revision '%s'", rev); Path cacheDir = getCacheDir() + "/nix/git"; if (!pathExists(cacheDir)) { - createDirs(cacheDir); runProgram("git", true, { "init", "--bare", cacheDir }); } @@ -228,6 +233,9 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); v.attrs->sort(); + + if (state.allowedPaths) + state.allowedPaths->insert(gitInfo.storePath); } static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index a317476c5829..5517d83df824 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -27,6 +27,9 @@ std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); HgInfo exportMercurial(ref<Store> store, const std::string & uri, std::string rev, const std::string & name) { + if (settings.pureEval && rev == "") + throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); + if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == ""; @@ -196,6 +199,9 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12)); mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount); v.attrs->sort(); + + if (state.allowedPaths) + state.allowedPaths->insert(hgInfo.storePath); } static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index d3aac6aba1ff..bcc05c2cdad6 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -37,6 +37,10 @@ MixCommonArgs::MixCommonArgs(const string & programName) std::string cat = "config"; settings.convertToArgs(*this, cat); + + // Backward compatibility hack: nix-env already had a --system flag. + if (programName == "nix-env") longFlags.erase("system"); + hiddenCategories.insert(cat); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 85d3c077ba5e..7d888202bbf1 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -193,9 +193,6 @@ LegacyArgs::LegacyArgs(const std::string & programName, mkFlag(0, "readonly-mode", "do not write to the Nix store", &settings.readOnlyMode); - mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors", - &settings.showTrace); - mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", &gcWarning, false); @@ -265,6 +262,7 @@ void printVersion(const string & programName) void showManPage(const string & name) { restoreSignals(); + setenv("MANPATH", settings.nixManDir.c_str(), 1); execlp("man", "man", name.c_str(), NULL); throw SysError(format("command 'man %1%' failed") % name.c_str()); } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 1dcc4f0ac942..8e4861232db5 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -22,6 +22,7 @@ public: int handleExceptions(const string & programName, std::function<void()> fun); +/* Don't forget to call initPlugins() after settings are initialized! */ void initNix(); void parseCmdLine(int argc, char * * argv, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ab971dd8b6d9..d1b278b8efbe 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -149,7 +149,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str /* Compress the NAR. */ narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); - auto narCompressed = compress(compression, *nar); + auto narCompressed = compress(compression, *nar, parallelCompression); auto now2 = std::chrono::steady_clock::now(); narInfo->fileHash = hashString(htSHA256, *narCompressed); narInfo->fileSize = narCompressed->size(); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 8492ff600eba..e20b968442b7 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -19,6 +19,8 @@ public: const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; + const Setting<bool> parallelCompression{this, false, "parallel-compression", + "enable multi-threading compression, available for xz only currently"}; private: diff --git a/src/libstore/build.cc b/src/libstore/build.cc index d4bd650baf22..1d611ffbaba5 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -6,6 +6,7 @@ #include "archive.hh" #include "affinity.hh" #include "builtins.hh" +#include "download.hh" #include "finally.hh" #include "compression.hh" #include "json.hh" @@ -48,7 +49,9 @@ #include <sys/param.h> #include <sys/mount.h> #include <sys/syscall.h> +#if HAVE_SECCOMP #include <seccomp.h> +#endif #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif @@ -1123,11 +1126,6 @@ void DerivationGoal::haveDerivation() return; } - /* Reject doing a hash build of anything other than a fixed-output - derivation. */ - if (buildMode == bmHash && !drv->isFixedOutput()) - throw Error("cannot do a hash build of non-fixed-output derivation '%1%'", drvPath); - /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ @@ -1319,9 +1317,7 @@ void DerivationGoal::inputsRealised() allPaths.insert(inputPaths.begin(), inputPaths.end()); /* Is this a fixed-output derivation? */ - fixedOutput = true; - for (auto & i : drv->outputs) - if (i.second.hash == "") fixedOutput = false; + fixedOutput = drv->isFixedOutput(); /* Don't repeat fixed-output derivations since they're already verified by their output hash.*/ @@ -1341,19 +1337,6 @@ void DerivationGoal::tryToBuild() { trace("trying to build"); - /* Check for the possibility that some other goal in this process - has locked the output since we checked in haveDerivation(). - (It can't happen between here and the lockPaths() call below - because we're not allowing multi-threading.) If so, put this - goal to sleep until another goal finishes, then try again. */ - for (auto & i : drv->outputs) - if (pathIsLockedByMe(worker.store.toRealPath(i.second.path))) { - debug(format("putting derivation '%1%' to sleep because '%2%' is locked by another goal") - % drvPath % i.second.path); - worker.waitForAnyGoal(shared_from_this()); - return; - } - /* Obtain locks on all output paths. The locks are automatically released when we exit this function or Nix crashes. If we can't acquire the lock, then continue; hopefully some other @@ -1777,6 +1760,19 @@ PathSet exportReferences(Store & store, PathSet storePaths) return paths; } +static std::once_flag dns_resolve_flag; + +static void preloadNSS() { + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a download of an invalid URL to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + std::call_once(dns_resolve_flag, []() { + DownloadRequest request("http://this.pre-initializes.the.dns.resolvers.invalid"); + request.tries = 1; // We only need to do it once, and this also suppresses an annoying warning + try { getDownloader()->download(request); } catch (...) {} + }); +} void DerivationGoal::startBuilder() { @@ -1787,6 +1783,9 @@ void DerivationGoal::startBuilder() % drv->platform % settings.thisSystem % drvPath); } + if (drv->isBuiltin()) + preloadNSS(); + #if __APPLE__ additionalSandboxProfile = get(drv->env, "__sandboxProfile"); #endif @@ -1810,8 +1809,13 @@ void DerivationGoal::startBuilder() useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1"; } - if (worker.store.storeDir != worker.store.realStoreDir) - useChroot = true; + if (worker.store.storeDir != worker.store.realStoreDir) { + #if __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif + } /* If `build-users-group' is not empty, then we have to build as one of the members of that group. */ @@ -2469,7 +2473,7 @@ void setupSeccomp() { #if __linux__ if (!settings.filterSyscalls) return; - +#if HAVE_SECCOMP scmp_filter_ctx ctx; if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) @@ -2515,6 +2519,11 @@ void setupSeccomp() if (seccomp_load(ctx) != 0) throw SysError("unable to load seccomp BPF program"); +#else + throw Error( + "seccomp is not supported on this platform; " + "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); +#endif #endif } @@ -2928,8 +2937,13 @@ void DerivationGoal::runChild() if (drv->isBuiltin()) { try { logger = makeJSONLogger(*logger); + + BasicDerivation drv2(*drv); + for (auto & e : drv2.env) + e.second = rewriteStrings(e.second, inputRewrites); + if (drv->builder == "builtin:fetchurl") - builtinFetchurl(*drv, netrcData); + builtinFetchurl(drv2, netrcData); else throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8)); _exit(0); @@ -2992,6 +3006,8 @@ void DerivationGoal::registerOutputs() bool runDiffHook = settings.runDiffHook; bool keepPreviousRound = settings.keepFailed || runDiffHook; + std::exception_ptr delayedException; + /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ @@ -3066,7 +3082,7 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - if (i.second.hash != "") { + if (fixedOutput) { bool recursive; Hash h; i.second.parseHashInfo(recursive, h); @@ -3082,27 +3098,34 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); - if (buildMode == bmHash) { - Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]); - printError(format("build produced path '%1%' with %2% hash '%3%'") - % dest % printHashType(h.type) % printHash16or32(h2)); - if (worker.store.isValidPath(dest)) - return; + + Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]); + + if (h != h2) { + + /* Throw an error after registering the path as + valid. */ + delayedException = std::make_exception_ptr( + BuildError("fixed-output derivation produced path '%s' with %s hash '%s' instead of the expected hash '%s'", + dest, printHashType(h.type), printHash16or32(h2), printHash16or32(h))); + Path actualDest = worker.store.toRealPath(dest); + + if (worker.store.isValidPath(dest)) + std::rethrow_exception(delayedException); + if (actualPath != actualDest) { PathLocks outputLocks({actualDest}); deletePath(actualDest); if (rename(actualPath.c_str(), actualDest.c_str()) == -1) throw SysError(format("moving '%1%' to '%2%'") % actualPath % dest); } + path = dest; actualPath = actualDest; - } else { - if (h != h2) - throw BuildError( - format("output path '%1%' has %2% hash '%3%' when '%4%' was expected") - % path % i.second.hashAlgo % printHash16or32(h2) % printHash16or32(h)); } + else + assert(path == dest); info.ca = makeFixedOutputCA(recursive, h2); } @@ -3279,6 +3302,11 @@ void DerivationGoal::registerOutputs() paths referenced by each of them. If there are cycles in the outputs, this will fail. */ worker.store.registerValidPaths(infos); + + /* In case of a fixed-output derivation hash mismatch, throw an + exception now that we have registered the output as valid. */ + if (delayedException) + std::rethrow_exception(delayedException); } @@ -3394,7 +3422,7 @@ void DerivationGoal::flushLine() else { if (settings.verboseBuild && (settings.printRepeatedBuilds || curRound == 1)) - printError(filterANSIEscapes(currentLogLine, true)); + printError(currentLogLine); else { logTail.push_back(currentLogLine); if (logTail.size() > settings.logLines) logTail.pop_front(); @@ -3636,7 +3664,7 @@ void SubstitutionGoal::tryNext() /* Update the total expected download size. */ auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info); - maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, narInfo->narSize); + maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize); maintainExpectedDownload = narInfo && narInfo->fileSize @@ -3650,9 +3678,12 @@ void SubstitutionGoal::tryNext() /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (worker.store.requireSigs && !info->checkSignatures(worker.store, worker.store.publicKeys)) { - printInfo(format("warning: substituter '%s' does not have a valid signature for path '%s'") - % sub->getUri() % storePath); + if (worker.store.requireSigs + && !sub->isTrusted + && !info->checkSignatures(worker.store, worker.store.publicKeys)) + { + printError("warning: substituter '%s' does not have a valid signature for path '%s'", + sub->getUri(), storePath); tryNext(); return; } @@ -3702,6 +3733,17 @@ void SubstitutionGoal::tryToRun() return; } + /* If the store path is already locked (probably by a + DerivationGoal), then put this goal to sleep. Note: we don't + acquire a lock here since that breaks addToStore(), so below we + handle an AlreadyLocked exception from addToStore(). The check + here is just an optimisation to prevent having to redo a + download due to a locked path. */ + if (pathIsLockedByMe(worker.store.toRealPath(storePath))) { + worker.waitForAWhile(shared_from_this()); + return; + } + maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); worker.updateProgress(); @@ -3718,7 +3760,7 @@ void SubstitutionGoal::tryToRun() PushActivity pact(act.id); copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()), - storePath, repair); + storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { @@ -3741,8 +3783,14 @@ void SubstitutionGoal::finished() try { promise.get_future().get(); + } catch (AlreadyLocked & e) { + /* Probably a DerivationGoal is already building this store + path. Sleep for a while and try again. */ + state = &SubstitutionGoal::init; + worker.waitForAWhile(shared_from_this()); + return; } catch (Error & e) { - printInfo(e.msg()); + printError(e.msg()); /* Try the next substitute. */ state = &SubstitutionGoal::tryNext; diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 4474dfd4b968..5ab625f42288 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -17,11 +17,13 @@ #include <curl/curl.h> -#include <queue> -#include <iostream> -#include <thread> +#include <algorithm> #include <cmath> +#include <cstring> +#include <iostream> +#include <queue> #include <random> +#include <thread> using namespace std::string_literals; @@ -91,6 +93,8 @@ struct CurlDownloader : public Downloader { if (!request.expectedETag.empty()) requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); + if (!request.mimeType.empty()) + requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); } ~DownloadItem() @@ -185,6 +189,22 @@ struct CurlDownloader : public Downloader return 0; } + size_t readOffset = 0; + int readCallback(char *buffer, size_t size, size_t nitems) + { + if (readOffset == request.data->length()) + return 0; + auto count = std::min(size * nitems, request.data->length() - readOffset); + memcpy(buffer, request.data->data() + readOffset, count); + readOffset += count; + return count; + } + + static int readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp) + { + return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); + } + long lowSpeedTimeout = 300; void init() @@ -225,6 +245,13 @@ struct CurlDownloader : public Downloader if (request.head) curl_easy_setopt(req, CURLOPT_NOBODY, 1); + if (request.data) { + curl_easy_setopt(req, CURLOPT_UPLOAD, 1L); + curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper); + curl_easy_setopt(req, CURLOPT_READDATA, this); + curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length()); + } + if (request.verifyTLS) { if (settings.caFile != "") curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); @@ -265,7 +292,7 @@ struct CurlDownloader : public Downloader } if (code == CURLE_OK && - (httpStatus == 200 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) + (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) { result.cached = httpStatus == 304; done = true; @@ -303,6 +330,7 @@ struct CurlDownloader : public Downloader // Don't bother retrying on certain cURL errors either switch (code) { case CURLE_FAILED_INIT: + case CURLE_URL_MALFORMAT: case CURLE_NOT_BUILT_IN: case CURLE_REMOTE_ACCESS_DENIED: case CURLE_FILE_COULDNT_READ_FILE: @@ -311,10 +339,10 @@ struct CurlDownloader : public Downloader case CURLE_BAD_FUNCTION_ARGUMENT: case CURLE_INTERFACE_FAILED: case CURLE_UNKNOWN_OPTION: - err = Misc; - break; + err = Misc; + break; default: // Shut up warnings - break; + break; } } @@ -689,7 +717,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}); + runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); } replaceSymlink(unpackedStorePath, unpackedLink); @@ -699,7 +727,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa if (expectedStorePath != "" && storePath != expectedStorePath) throw nix::Error("store path mismatch in file downloaded from '%s'", url); - return storePath; + return store->toRealPath(storePath); } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index f2d65ad8d61d..d9d525d4e65f 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -18,6 +18,8 @@ struct DownloadRequest unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; + std::shared_ptr<std::string> data; + std::string mimeType; DownloadRequest(const std::string & uri) : uri(uri), parentAct(curActivity) { } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ab2c5ca0274c..943b16c28fa3 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -324,10 +324,8 @@ Roots LocalStore::findRootsNoTemp() { Roots roots; - /* Process direct roots in {gcroots,manifests,profiles}. */ + /* Process direct roots in {gcroots,profiles}. */ findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); - if (pathExists(stateDir + "/manifests")) - findRoots(stateDir + "/manifests", DT_UNKNOWN, roots); findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); /* Add additional roots returned by the program specified by the diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index d3c96ddd6e61..f46e8326235f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -6,6 +6,7 @@ #include <algorithm> #include <map> #include <thread> +#include <dlfcn.h> namespace nix { @@ -37,6 +38,7 @@ Settings::Settings() , nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))) , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))) , nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))) + , nixManDir(canonPath(NIX_MAN_DIR)) , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; @@ -137,4 +139,46 @@ void MaxBuildJobsSetting::set(const std::string & str) throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); } + +void initPlugins() +{ + for (const auto & pluginFile : settings.pluginFiles.get()) { + Paths pluginFiles; + try { + auto ents = readDirectory(pluginFile); + for (const auto & ent : ents) + pluginFiles.emplace_back(pluginFile + "/" + ent.name); + } catch (SysError & e) { + if (e.errNo != ENOTDIR) + throw; + pluginFiles.emplace_back(pluginFile); + } + for (const auto & file : pluginFiles) { + /* handle is purposefully leaked as there may be state in the + DSO needed by the action of the plugin. */ + void *handle = + dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) + throw Error("could not dynamically open plugin file '%s%': %s%", file, dlerror()); + } + } + /* We handle settings registrations here, since plugins can add settings */ + if (RegisterSetting::settingRegistrations) { + for (auto & registration : *RegisterSetting::settingRegistrations) + settings.addSetting(registration); + delete RegisterSetting::settingRegistrations; + } + settings.handleUnknownSettings(); +} + +RegisterSetting::SettingRegistrations * RegisterSetting::settingRegistrations; + +RegisterSetting::RegisterSetting(AbstractSetting * s) +{ + if (!settingRegistrations) + settingRegistrations = new SettingRegistrations; + settingRegistrations->emplace_back(s); +} + + } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 5c857cbb6a9c..dd01f832df0c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -82,6 +82,9 @@ public: /* The directory where the main programs are stored. */ Path nixBinDir; + /* The directory where the man pages are stored. */ + Path nixManDir; + /* File name of the socket the daemon listens to. */ Path nixDaemonSocketFile; @@ -138,6 +141,11 @@ public: Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders", "A semicolon-separated list of build machines, in the format of nix.machines."}; + Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes", + "Whether build machines should use their own substitutes for obtaining " + "build dependencies if possible, rather than waiting for this host to " + "upload them."}; + Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", "Amount of reserved disk space for the garbage collector."}; @@ -150,7 +158,7 @@ public: Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering", "Whether to call sync() before registering a path as valid."}; - Setting<bool> useSubstitutes{this, true, "use-substitutes", + Setting<bool> useSubstitutes{this, true, "substitute", "Whether to use substitutes.", {"build-use-substitutes"}}; @@ -206,7 +214,8 @@ public: bool lockCPU; /* Whether to show a stack trace if Nix evaluation fails. */ - bool showTrace = false; + Setting<bool> showTrace{this, false, "show-trace", + "Whether to show a stack trace on evaluation errors."}; Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", "Whether builtin functions that allow executing native code should be enabled."}; @@ -227,6 +236,9 @@ public: "Whether to restrict file system access to paths in $NIX_PATH, " "and network access to the URI prefixes listed in 'allowed-uris'."}; + Setting<bool> pureEval{this, false, "pure-eval", + "Whether to restrict file system and network access to files specified by cryptographic hash."}; + Setting<size_t> buildRepeat{this, 0, "repeat", "The number of times to repeat a build in order to verify determinism.", {"build-repeat"}}; @@ -278,10 +290,7 @@ public: Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."}; - Setting<std::string> signedBinaryCaches{this, "*", "signed-binary-caches", - "Obsolete."}; - - Setting<bool> requireSigs{this, signedBinaryCaches == "*", "require-sigs", + Setting<bool> requireSigs{this, true, "require-sigs", "Whether to check that any non-content-addressed path added to the " "Nix store has a valid signature (that is, one signed using a key " "listed in 'trusted-public-keys'."}; @@ -361,14 +370,28 @@ public: Setting<Strings> allowedUris{this, {}, "allowed-uris", "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + + Setting<Paths> pluginFiles{this, {}, "plugin-files", + "Plugins to dynamically load at nix initialization time."}; }; // FIXME: don't use a global variable. extern Settings settings; +/* This should be called after settings are initialized, but before + anything else */ +void initPlugins(); + extern const string nixVersion; +struct RegisterSetting +{ + typedef std::vector<AbstractSetting *> SettingRegistrations; + static SettingRegistrations * settingRegistrations; + RegisterSetting(AbstractSetting * s); +}; + } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 057337685791..b9e9cd5daba5 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -38,7 +38,7 @@ public: try { BinaryCacheStore::init(); } catch (UploadToHTTP &) { - throw Error(format("'%s' does not appear to be a binary cache") % cacheUri); + throw Error("'%s' does not appear to be a binary cache", cacheUri); } diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority); } @@ -67,7 +67,14 @@ protected: const std::string & data, const std::string & mimeType) override { - throw UploadToHTTP("uploading to an HTTP binary cache is not supported"); + auto req = DownloadRequest(cacheUri + "/" + path); + req.data = std::make_shared<string>(data); // FIXME: inefficient + req.mimeType = mimeType; + try { + getDownloader()->download(req); + } catch (DownloadError & e) { + throw UploadToHTTP(format("uploading to HTTP binary cache at %1% not supported: %2%") % cacheUri % e.msg()); + } } void getFile(const std::string & path, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7afecc1cfc62..4afe51ea91ec 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -992,8 +992,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & /* Lock the output path. But don't lock if we're being called from a build hook (whose parent process already acquired a lock on this path). */ - Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS")); - if (find(locksHeld.begin(), locksHeld.end(), info.path) == locksHeld.end()) + if (!locksHeld.count(info.path)) outputLock.lockPaths({realPath}); if (repair || !isValidPath(info.path)) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index d35cd1a949eb..bbd50e1c1451 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -19,7 +19,7 @@ namespace nix { /* Nix store and database schema version. Version 1 (or 0) was Nix <= 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is - Nix 1.0. Version 7 is Nix 1.3. Version 10 is 1.12. */ + Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ const int nixSchemaVersion = 10; @@ -104,6 +104,9 @@ private: public: + // Hack for build-remote.cc. + PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS")); + /* Initialise the local store, upgrading the schema if necessary. */ LocalStore(const Params & params); diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 50c46ce6fe99..e11efa5c2b54 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -9,6 +9,9 @@ libstore_SOURCES := $(wildcard $(d)/*.cc) libstore_LIBS = libutil libformat libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread +ifneq ($(OS), FreeBSD) + libstore_LDFLAGS += -ldl +endif libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb @@ -22,7 +25,7 @@ ifeq ($(OS), SunOS) libstore_LDFLAGS += -lsocket endif -ifeq ($(OS), Linux) +ifeq ($(HAVE_SECCOMP), 1) libstore_LDFLAGS += -lseccomp endif @@ -35,6 +38,7 @@ libstore_CXXFLAGS = \ -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ -DNIX_BIN_DIR=\"$(bindir)\" \ + -DNIX_MAN_DIR=\"$(mandir)\" \ -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \ -DLSOF=\"$(lsof)\" diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 1903355a236e..2871199de16e 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -1,5 +1,7 @@ #pragma once +#include <functional> + #include "fs-accessor.hh" namespace nix { diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 587f29598851..08d1efdbeb01 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -113,8 +113,10 @@ bool PathLocks::lockPaths(const PathSet & _paths, { auto lockedPaths(lockedPaths_.lock()); - if (lockedPaths->count(lockPath)) - throw Error("deadlock: trying to re-acquire self-held lock '%s'", lockPath); + if (lockedPaths->count(lockPath)) { + if (!wait) return false; + throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath); + } lockedPaths->insert(lockPath); } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 2a7de611446e..db51f950a320 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -2,10 +2,8 @@ #include "util.hh" - namespace nix { - /* Open (possibly create) a lock file and return the file descriptor. -1 is returned if create is false and the lock could not be opened because it doesn't exist. Any other error throws an exception. */ @@ -18,6 +16,7 @@ enum LockType { ltRead, ltWrite, ltNone }; bool lockFile(int fd, LockType lockType, bool wait); +MakeError(AlreadyLocked, Error); class PathLocks { @@ -38,9 +37,6 @@ public: void setDeletion(bool deletePaths); }; - -// FIXME: not thread-safe! bool pathIsLockedByMe(const Path & path); - } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 77b41b6bf8a8..8f0b65557ac4 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -78,9 +78,22 @@ UDSRemoteStore::UDSRemoteStore(const Params & params) } +UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params) + : Store(params) + , LocalFSStore(params) + , RemoteStore(params) + , path(socket_path) +{ +} + + std::string UDSRemoteStore::getUri() { - return "daemon"; + if (path) { + return std::string("unix://") + *path; + } else { + return "daemon"; + } } @@ -98,7 +111,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection() throw SysError("cannot create Unix domain socket"); closeOnExec(conn->fd.get()); - string socketPath = settings.nixDaemonSocketFile; + string socketPath = path ? *path : settings.nixDaemonSocketFile; struct sockaddr_un addr; addr.sun_family = AF_UNIX; @@ -721,5 +734,14 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source) } } +static std::string uriScheme = "unix://"; + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr<Store> +{ + if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; + return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params); +}); } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 30c6beae6ff2..7f36e206416b 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -134,6 +134,7 @@ class UDSRemoteStore : public LocalFSStore, public RemoteStore public: UDSRemoteStore(const Params & params); + UDSRemoteStore(std::string path, const Params & params); std::string getUri() override; @@ -145,6 +146,7 @@ private: }; ref<RemoteStore::Connection> openConnection() override; + std::experimental::optional<std::string> path; }; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 0079da1becfb..23af452094cf 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -10,6 +10,7 @@ #include "istringstream_nocopy.hh" #include <aws/core/Aws.h> +#include <aws/core/VersionConfig.h> #include <aws/core/auth/AWSCredentialsProvider.h> #include <aws/core/auth/AWSCredentialsProviderChain.h> #include <aws/core/client/ClientConfiguration.h> @@ -87,7 +88,14 @@ S3Helper::S3Helper(const std::string & profile, const std::string & region) std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>()) : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())), - *config, true, false)) + *config, + // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 +#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3 + false, +#else + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, +#endif + false)) { } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 77ab87ef728e..8830edcc3449 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -222,11 +222,10 @@ Path Store::makeTextPath(const string & name, const Hash & hash, } -std::pair<Path, Hash> Store::computeStorePathForPath(const Path & srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter) const +std::pair<Path, Hash> Store::computeStorePathForPath(const string & name, + const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const { Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - string name = baseNameOf(srcPath); Path dstPath = makeFixedOutputPath(recursive, h, name); return std::pair<Path, Hash>(dstPath, h); } @@ -840,7 +839,7 @@ ref<Store> openStore(const std::string & uri_, for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); if (store) { - store->warnUnknownSettings(); + store->handleUnknownSettings(); return ref<Store>(store); } } @@ -897,7 +896,11 @@ std::list<ref<Store>> getDefaultSubstituters() auto addStore = [&](const std::string & uri) { if (done.count(uri)) return; done.insert(uri); - stores.push_back(openStore(uri)); + try { + stores.push_back(openStore(uri)); + } catch (Error & e) { + printError("warning: %s", e.what()); + } }; for (auto uri : settings.substituters.get()) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index d1e1b5d6f452..563aa566bd37 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -192,7 +192,7 @@ struct ValidPathInfo typedef list<ValidPathInfo> ValidPathInfos; -enum BuildMode { bmNormal, bmRepair, bmCheck, bmHash }; +enum BuildMode { bmNormal, bmRepair, bmCheck }; struct BuildResult @@ -248,6 +248,8 @@ public: const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; + const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; + protected: struct State @@ -305,9 +307,9 @@ public: /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ - std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter) const; + std::pair<Path, Hash> computeStorePathForPath(const string & name, + const Path & srcPath, bool recursive = true, + HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; /* Preparatory part of addTextToStore(). @@ -597,6 +599,11 @@ public: "nix-cache-info" file. Lower value means higher priority. */ virtual int getPriority() { return 0; } + virtual Path toRealPath(const Path & storePath) + { + return storePath; + } + protected: Stats stats; @@ -639,9 +646,10 @@ public: virtual Path getRealStoreDir() { return storeDir; } - Path toRealPath(const Path & storePath) + Path toRealPath(const Path & storePath) override { - return getRealStoreDir() + "/" + baseNameOf(storePath); + assert(isInStore(storePath)); + return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); } std::shared_ptr<std::string> getBuildLog(const Path & path) override; @@ -699,6 +707,9 @@ void removeTempRoots(); * โdaemonโ: The Nix store accessed via a Unix domain socket connection to nix-daemon. + * โunix://<path>โ: The Nix store accessed via a Unix domain socket + connection to nix-daemon, with the socket located at <path>. + * โautoโ or โโ: Equivalent to โlocalโ or โdaemonโ depending on whether the user has write access to the local Nix store/database. diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 2b3dff3a5ea1..470c925ed7a6 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,12 +1,18 @@ #include "compression.hh" #include "util.hh" #include "finally.hh" +#include "logging.hh" #include <lzma.h> #include <bzlib.h> #include <cstdio> #include <cstring> +#if HAVE_BROTLI +#include <brotli/decode.h> +#include <brotli/encode.h> +#endif // HAVE_BROTLI + #include <iostream> namespace nix { @@ -94,14 +100,62 @@ static ref<std::string> decompressBzip2(const std::string & in) static ref<std::string> decompressBrotli(const std::string & in) { - // FIXME: use libbrotli - return make_ref<std::string>(runProgram(BRO, true, {"-d"}, {in})); +#if !HAVE_BROTLI + return make_ref<std::string>(runProgram(BROTLI, true, {"-d"}, {in})); +#else + auto *s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + if (!s) + throw CompressionError("unable to initialize brotli decoder"); + + Finally free([s]() { BrotliDecoderDestroyInstance(s); }); + + uint8_t outbuf[BUFSIZ]; + ref<std::string> res = make_ref<std::string>(); + const uint8_t *next_in = (uint8_t *)in.c_str(); + size_t avail_in = in.size(); + uint8_t *next_out = outbuf; + size_t avail_out = sizeof(outbuf); + + while (true) { + checkInterrupt(); + + auto ret = BrotliDecoderDecompressStream(s, + &avail_in, &next_in, + &avail_out, &next_out, + nullptr); + + switch (ret) { + case BROTLI_DECODER_RESULT_ERROR: + throw CompressionError("error while decompressing brotli file"); + case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: + throw CompressionError("incomplete or corrupt brotli file"); + case BROTLI_DECODER_RESULT_SUCCESS: + if (avail_in != 0) + throw CompressionError("unexpected input after brotli decompression"); + break; + case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: + // I'm not sure if this can happen, but abort if this happens with empty buffer + if (avail_out == sizeof(outbuf)) + throw CompressionError("brotli decompression requires larger buffer"); + break; + } + + // Always ensure we have full buffer for next invocation + if (avail_out < sizeof(outbuf)) { + res->append((char*)outbuf, sizeof(outbuf) - avail_out); + next_out = outbuf; + avail_out = sizeof(outbuf); + } + + if (ret == BROTLI_DECODER_RESULT_SUCCESS) return res; + } +#endif // HAVE_BROTLI } -ref<std::string> compress(const std::string & method, const std::string & in) +ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel) { StringSink ssink; - auto sink = makeCompressionSink(method, ssink); + auto sink = makeCompressionSink(method, ssink, parallel); (*sink)(in); sink->finish(); return ssink.s; @@ -136,10 +190,9 @@ struct XzSink : CompressionSink lzma_stream strm = LZMA_STREAM_INIT; bool finished = false; - XzSink(Sink & nextSink) : nextSink(nextSink) - { - lzma_ret ret = lzma_easy_encoder( - &strm, 6, LZMA_CHECK_CRC64); + template <typename F> + XzSink(Sink & nextSink, F&& initEncoder) : nextSink(nextSink) { + lzma_ret ret = initEncoder(); if (ret != LZMA_OK) throw CompressionError("unable to initialise lzma encoder"); // FIXME: apply the x86 BCJ filter? @@ -147,6 +200,9 @@ struct XzSink : CompressionSink strm.next_out = outbuf; strm.avail_out = sizeof(outbuf); } + XzSink(Sink & nextSink) : XzSink(nextSink, [this]() { + return lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); + }) {} ~XzSink() { @@ -200,6 +256,27 @@ struct XzSink : CompressionSink } }; +#ifdef HAVE_LZMA_MT +struct ParallelXzSink : public XzSink +{ + ParallelXzSink(Sink &nextSink) : XzSink(nextSink, [this]() { + lzma_mt mt_options = {}; + mt_options.flags = 0; + mt_options.timeout = 300; // Using the same setting as the xz cmd line + mt_options.preset = LZMA_PRESET_DEFAULT; + mt_options.filters = NULL; + mt_options.check = LZMA_CHECK_CRC64; + mt_options.threads = lzma_cputhreads(); + mt_options.block_size = 0; + if (mt_options.threads == 0) + mt_options.threads = 1; + // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the + // number of threads. + return lzma_stream_encoder_mt(&strm, &mt_options); + }) {} +}; +#endif + struct BzipSink : CompressionSink { Sink & nextSink; @@ -270,36 +347,142 @@ struct BzipSink : CompressionSink } }; -struct BrotliSink : CompressionSink +struct LambdaCompressionSink : CompressionSink { Sink & nextSink; std::string data; + using CompressFnTy = std::function<std::string(const std::string&)>; + CompressFnTy compressFn; + LambdaCompressionSink(Sink& nextSink, CompressFnTy compressFn) + : nextSink(nextSink) + , compressFn(std::move(compressFn)) + { + }; + + void finish() override + { + flush(); + nextSink(compressFn(data)); + } + + void write(const unsigned char * data, size_t len) override + { + checkInterrupt(); + this->data.append((const char *) data, len); + } +}; + +struct BrotliCmdSink : LambdaCompressionSink +{ + BrotliCmdSink(Sink& nextSink) + : LambdaCompressionSink(nextSink, [](const std::string& data) { + return runProgram(BROTLI, true, {}, data); + }) + { + } +}; + +#if HAVE_BROTLI +struct BrotliSink : CompressionSink +{ + Sink & nextSink; + uint8_t outbuf[BUFSIZ]; + BrotliEncoderState *state; + bool finished = false; BrotliSink(Sink & nextSink) : nextSink(nextSink) { + state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); + if (!state) + throw CompressionError("unable to initialise brotli encoder"); } ~BrotliSink() { + BrotliEncoderDestroyInstance(state); } - // FIXME: use libbrotli - void finish() override { flush(); - nextSink(runProgram(BRO, true, {}, data)); + assert(!finished); + + const uint8_t *next_in = nullptr; + size_t avail_in = 0; + uint8_t *next_out = outbuf; + size_t avail_out = sizeof(outbuf); + while (!finished) { + checkInterrupt(); + + if (!BrotliEncoderCompressStream(state, + BROTLI_OPERATION_FINISH, + &avail_in, &next_in, + &avail_out, &next_out, + nullptr)) + throw CompressionError("error while finishing brotli file"); + + finished = BrotliEncoderIsFinished(state); + if (avail_out == 0 || finished) { + nextSink(outbuf, sizeof(outbuf) - avail_out); + next_out = outbuf; + avail_out = sizeof(outbuf); + } + } } void write(const unsigned char * data, size_t len) override { - checkInterrupt(); - this->data.append((const char *) data, len); + assert(!finished); + + // Don't feed brotli too much at once + const size_t CHUNK_SIZE = sizeof(outbuf) << 2; + while (len) { + size_t n = std::min(CHUNK_SIZE, len); + writeInternal(data, n); + data += n; + len -= n; + } + } + private: + void writeInternal(const unsigned char * data, size_t len) + { + assert(!finished); + + const uint8_t *next_in = data; + size_t avail_in = len; + uint8_t *next_out = outbuf; + size_t avail_out = sizeof(outbuf); + + while (avail_in > 0) { + checkInterrupt(); + + if (!BrotliEncoderCompressStream(state, + BROTLI_OPERATION_PROCESS, + &avail_in, &next_in, + &avail_out, &next_out, + nullptr)) + throw CompressionError("error while compressing brotli file"); + + if (avail_out < sizeof(outbuf) || avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - avail_out); + next_out = outbuf; + avail_out = sizeof(outbuf); + } + } } }; +#endif // HAVE_BROTLI -ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink) +ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) { + if (parallel) { +#ifdef HAVE_LZMA_MT + if (method == "xz") + return make_ref<ParallelXzSink>(nextSink); +#endif + printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method); + } + if (method == "none") return make_ref<NoneSink>(nextSink); else if (method == "xz") @@ -307,7 +490,11 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next else if (method == "bzip2") return make_ref<BzipSink>(nextSink); else if (method == "br") +#if HAVE_BROTLI return make_ref<BrotliSink>(nextSink); +#else + return make_ref<BrotliCmdSink>(nextSink); +#endif else throw UnknownCompressionMethod(format("unknown compression method '%s'") % method); } diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index e3e6f5a99303..a0d7530d74fc 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -8,7 +8,7 @@ namespace nix { -ref<std::string> compress(const std::string & method, const std::string & in); +ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false); ref<std::string> decompress(const std::string & method, const std::string & in); @@ -17,7 +17,7 @@ struct CompressionSink : BufferedSink virtual void finish() = 0; }; -ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink); +ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false); MakeError(UnknownCompressionMethod, Error); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index d46ca65a3863..ce6858f0d65a 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -7,10 +7,12 @@ namespace nix { void Config::set(const std::string & name, const std::string & value) { auto i = _settings.find(name); - if (i == _settings.end()) - throw UsageError("unknown setting '%s'", name); - i->second.setting->set(value); - i->second.setting->overriden = true; + if (i == _settings.end()) { + extras.emplace(name, value); + } else { + i->second.setting->set(value); + i->second.setting->overriden = true; + } } void Config::addSetting(AbstractSetting * setting) @@ -21,34 +23,34 @@ void Config::addSetting(AbstractSetting * setting) bool set = false; - auto i = initials.find(setting->name); - if (i != initials.end()) { + auto i = extras.find(setting->name); + if (i != extras.end()) { setting->set(i->second); setting->overriden = true; - initials.erase(i); + extras.erase(i); set = true; } for (auto & alias : setting->aliases) { - auto i = initials.find(alias); - if (i != initials.end()) { + auto i = extras.find(alias); + if (i != extras.end()) { if (set) warn("setting '%s' is set, but it's an alias of '%s' which is also set", alias, setting->name); else { setting->set(i->second); setting->overriden = true; - initials.erase(i); + extras.erase(i); set = true; } } } } -void Config::warnUnknownSettings() +void Config::handleUnknownSettings() { - for (auto & i : initials) - warn("unknown setting '%s'", i.first); + for (auto & s : extras) + warn("unknown setting '%s'", s.first); } StringMap Config::getSettings(bool overridenOnly) @@ -60,7 +62,7 @@ StringMap Config::getSettings(bool overridenOnly) return res; } -void Config::applyConfigFile(const Path & path, bool fatal) +void Config::applyConfigFile(const Path & path) { try { string contents = readFile(path); @@ -80,7 +82,31 @@ void Config::applyConfigFile(const Path & path, bool fatal) vector<string> tokens = tokenizeString<vector<string> >(line); if (tokens.empty()) continue; - if (tokens.size() < 2 || tokens[1] != "=") + if (tokens.size() < 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + + auto include = false; + auto ignoreMissing = false; + if (tokens[0] == "include") + include = true; + else if (tokens[0] == "!include") { + include = true; + ignoreMissing = true; + } + + if (include) { + if (tokens.size() != 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + auto p = absPath(tokens[1], dirOf(path)); + if (pathExists(p)) { + applyConfigFile(p); + } else if (!ignoreMissing) { + throw Error("file '%1%' included from '%2%' not found", p, path); + } + continue; + } + + if (tokens[1] != "=") throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); string name = tokens[0]; @@ -88,12 +114,7 @@ void Config::applyConfigFile(const Path & path, bool fatal) vector<string>::iterator i = tokens.begin(); advance(i, 2); - try { - set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow - } catch (UsageError & e) { - if (fatal) throw; - warn("in configuration file '%s': %s", path, e.what()); - } + set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow }; } catch (SysError &) { } } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 9a32af528ec7..d2e7faf17434 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -48,25 +48,25 @@ private: Settings _settings; - StringMap initials; + StringMap extras; public: Config(const StringMap & initials) - : initials(initials) + : extras(initials) { } void set(const std::string & name, const std::string & value); void addSetting(AbstractSetting * setting); - void warnUnknownSettings(); + void handleUnknownSettings(); StringMap getSettings(bool overridenOnly = false); const Settings & _getSettings() { return _settings; } - void applyConfigFile(const Path & path, bool fatal = false); + void applyConfigFile(const Path & path); void resetOverriden(); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 11e3c9dca58a..75e4767550f7 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -189,7 +189,8 @@ Hash::Hash(const std::string & s, HashType type) else if (size == base64Len()) { auto d = base64Decode(std::string(s, pos)); - assert(d.size() == hashSize); + if (d.size() != hashSize) + throw BadHash("invalid base-64 hash '%s'", s); memcpy(hash, d.data(), hashSize); } diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 0721b21c2089..5fc2aab569da 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,8 +6,8 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) +libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) libutil_LIBS = libformat -libutil_CXXFLAGS = -DBRO=\"$(bro)\" +libutil_CXXFLAGS = -DBROTLI=\"$(brotli)\" diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 6924e0080475..27a631a37d10 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -44,7 +44,7 @@ public: prefix = std::string("<") + c + ">"; } - writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n"); + writeToStderr(prefix + filterANSIEscapes(fs.s) + "\n"); } void startActivity(ActivityId act, Verbosity lvl, ActivityType type, diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index e0ec66c01803..5ee0b88ef50f 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -21,13 +21,29 @@ public: MonitorFdHup(int fd) { thread = std::thread([fd]() { - /* Wait indefinitely until a POLLHUP occurs. */ - struct pollfd fds[1]; - fds[0].fd = fd; - fds[0].events = 0; - if (poll(fds, 1, -1) == -1) abort(); // can't happen - assert(fds[0].revents & POLLHUP); - triggerInterrupt(); + while (true) { + /* Wait indefinitely until a POLLHUP occurs. */ + struct pollfd fds[1]; + fds[0].fd = fd; + /* This shouldn't be necessary, but macOS doesn't seem to + like a zeroed out events field. + See rdar://37537852. + */ + fds[0].events = POLLHUP; + auto count = poll(fds, 1, -1); + if (count == -1) abort(); // can't happen + /* This shouldn't happen, but can on macOS due to a bug. + See rdar://37550628. + + This may eventually need a delay or further + coordination with the main thread if spinning proves + too harmful. + */ + if (count == 0) continue; + assert(fds[0].revents & POLLHUP); + triggerInterrupt(); + break; + } }); }; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 950e6362a245..9e2a502afaf8 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -67,7 +67,8 @@ void FdSink::write(const unsigned char * data, size_t len) try { writeFull(fd, data, len); } catch (SysError & e) { - _good = true; + _good = false; + throw; } } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 197df0c44aa0..2391e14a94bd 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -73,6 +73,13 @@ std::map<std::string, std::string> getEnv() } +void clearEnv() +{ + for (auto & name : getEnv()) + unsetenv(name.first.c_str()); +} + + Path absPath(Path path, Path dir) { if (path[0] != '/') { @@ -192,6 +199,12 @@ bool isInDir(const Path & path, const Path & dir) } +bool isDirOrInDir(const Path & path, const Path & dir) +{ + return path == dir or isInDir(path, dir); +} + + struct stat lstat(const Path & path) { struct stat st; @@ -1172,36 +1185,51 @@ void ignoreException() } -string filterANSIEscapes(const string & s, bool nixOnly) +std::string filterANSIEscapes(const std::string & s, unsigned int width) { - string t, r; - enum { stTop, stEscape, stCSI } state = stTop; - for (auto c : s) { - if (state == stTop) { - if (c == '\e') { - state = stEscape; - r = c; - } else - t += c; - } else if (state == stEscape) { - r += c; - if (c == '[') - state = stCSI; - else { - t += r; - state = stTop; + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; } - } else { - r += c; - if (c >= 0x40 && c <= 0x7e) { - if (nixOnly && (c != 'p' && c != 'q' && c != 's' && c != 'a' && c != 'b')) - t += r; - state = stTop; - r.clear(); + + if (last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; } } + + else if (*i == '\r') + // do nothing for now + i++; + + else { + t += *i++; w++; + } } - t += r; + return t; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index a3494e09b09b..c5c537ee63d8 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -32,6 +32,9 @@ string getEnv(const string & key, const string & def = ""); /* Get the entire environment. */ std::map<std::string, std::string> getEnv(); +/* Clear the environment. */ +void clearEnv(); + /* Return an absolutized path, resolving paths relative to the specified directory, or the current directory otherwise. The path is also canonicalised. */ @@ -53,10 +56,12 @@ Path dirOf(const Path & path); following the final `/'. */ string baseNameOf(const Path & path); -/* Check whether a given path is a descendant of the given - directory. */ +/* Check whether 'path' is a descendant of 'dir'. */ bool isInDir(const Path & path, const Path & dir); +/* Check whether 'path' is equal to 'dir' or a descendant of 'dir'. */ +bool isDirOrInDir(const Path & path, const Path & dir); + /* Get status of `path'. */ struct stat lstat(const Path & path); @@ -386,10 +391,12 @@ void ignoreException(); #define ANSI_BLUE "\e[34;1m" -/* Filter out ANSI escape codes from the given string. If โnixOnlyโ is - set, only filter escape codes generated by Nixpkgs' stdenv (used to - denote nesting etc.). */ -string filterANSIEscapes(const string & s, bool nixOnly = false); +/* Truncate a string to 'width' printable characters. Certain ANSI + escape sequences (such as colour setting) are copied but not + included in the character count. Other ANSI escape sequences are + filtered. Also, tabs are expanded to spaces. */ +std::string filterANSIEscapes(const std::string & s, + unsigned int width = std::numeric_limits<unsigned int>::max()); /* Base64 encoding/decoding. */ diff --git a/src/linenoise/linenoise.cpp b/src/linenoise/linenoise.cpp index 8ee8984d6948..c57505d2fa97 100644 --- a/src/linenoise/linenoise.cpp +++ b/src/linenoise/linenoise.cpp @@ -2587,13 +2587,6 @@ int InputBuffer::getInputLine(PromptBase& pi) { // ctrl-I/tab, command completion, needs to be before switch statement if (c == ctrlChar('I') && completionCallback) { - if (pos == 0) // SERVER-4967 -- in earlier versions, you could paste - // previous output - continue; // back into the shell ... this output may have leading - // tabs. - // This hack (i.e. what the old code did) prevents command completion - // on an empty line but lets users paste text with leading tabs. - killRing.lastAction = KillRing::actionOther; historyRecallMostRecent = false; diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 58366daa6e86..99f773451ffe 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -141,7 +141,7 @@ void mainWrapped(int argc, char * * argv) else if (*arg == "--version") printVersion(myName); - else if (*arg == "--add-drv-link") + else if (*arg == "--add-drv-link" || *arg == "--indirect") ; // obsolete else if (*arg == "--no-out-link" || *arg == "--no-link") @@ -167,9 +167,6 @@ void mainWrapped(int argc, char * * argv) buildMode = bmRepair; } - else if (*arg == "--hash") - buildMode = bmHash; - else if (*arg == "--run-env") // obsolete runEnv = true; @@ -235,6 +232,8 @@ void mainWrapped(int argc, char * * argv) myArgs.parseCmdline(args); + initPlugins(); + if (packages && fromArgs) throw UsageError("'-p' and '-E' are mutually exclusive"); @@ -279,8 +278,8 @@ void mainWrapped(int argc, char * * argv) else /* If we're in a #! script, interpret filenames relative to the script. */ - exprs.push_back(state.parseExprFromFile(resolveExprPath(lookupFileArg(state, - inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))); + exprs.push_back(state.parseExprFromFile(resolveExprPath(state.checkSourcePath(lookupFileArg(state, + inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); } /* Evaluate them into derivations. */ diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 370f216abccd..ec9a7174ecb9 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -213,6 +213,9 @@ int main(int argc, char ** argv) } return true; }); + + initPlugins(); + switch (cmd) { case cNone: throw UsageError("no command specified"); diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index cc663a96924d..37fe22f48134 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -77,6 +77,8 @@ int main(int argc, char * * argv) return true; }); + initPlugins(); + auto profilesDir = settings.nixStateDir + "/profiles"; if (removeOld) removeOldGenerations(profilesDir); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 861fc2e5cd64..dfb1b8fc5dc4 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -44,6 +44,8 @@ int main(int argc, char ** argv) return true; }); + initPlugins(); + if (sshHost.empty()) throw UsageError("no host name specified"); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 5629cc64b96e..890bffa19aa5 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -411,7 +411,7 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store, /* Repairing is not atomic, so disallowed for "untrusted" clients. */ if (mode == bmRepair && !trusted) - throw Error("repairing is not supported when building through the Nix daemon"); + throw Error("repairing is not allowed because you are not in 'trusted-users'"); } logger->startWork(); store->buildPaths(drvs, mode); @@ -994,7 +994,7 @@ static void daemonLoop(char * * argv) if (matchUser(user, group, trustedUsers)) trusted = true; - if (!trusted && !matchUser(user, group, allowedUsers)) + if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) throw Error(format("user '%1%' is not allowed to connect to the Nix daemon") % user); printInfo(format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : "")) @@ -1060,6 +1060,8 @@ int main(int argc, char * * argv) return true; }); + initPlugins(); + if (stdio) { if (getStoreType() == tDaemon) { /* Forward on this connection to the real daemon */ diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 016caf6d2346..97e66cbd937e 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1393,6 +1393,8 @@ int main(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); + initPlugins(); + if (!op) throw UsageError("no operation specified"); auto store = openStore(); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 55ac007e8682..dd262bea0918 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -151,6 +151,8 @@ int main(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); + initPlugins(); + if (evalOnly && !wantsReadWrite) settings.readOnlyMode = true; @@ -182,7 +184,7 @@ int main(int argc, char * * argv) for (auto & i : files) { Expr * e = fromArgs ? state.parseExprFromString(i, absPath(".")) - : state.parseExprFromFile(resolveExprPath(lookupFileArg(state, i))); + : state.parseExprFromFile(resolveExprPath(state.checkSourcePath(lookupFileArg(state, i)))); processExpr(state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index fef3eaa45538..fa7ee254500c 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -89,6 +89,8 @@ int main(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); + initPlugins(); + if (args.size() > 2) throw UsageError("too many arguments"); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f6f276dd1798..e1e27ceef94d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -122,7 +122,6 @@ static void opRealise(Strings opFlags, Strings opArgs) if (i == "--dry-run") dryRun = true; else if (i == "--repair") buildMode = bmRepair; else if (i == "--check") buildMode = bmCheck; - else if (i == "--hash") buildMode = bmHash; else if (i == "--ignore-unknown") ignoreUnknown = true; else throw UsageError(format("unknown flag '%1%'") % i); @@ -632,6 +631,7 @@ static void opDump(Strings opFlags, Strings opArgs) FdSink sink(STDOUT_FILENO); string path = *opArgs.begin(); dumpPath(path, sink); + sink.flush(); } @@ -657,6 +657,7 @@ static void opExport(Strings opFlags, Strings opArgs) FdSink sink(STDOUT_FILENO); store->exportPaths(opArgs, sink); + sink.flush(); } @@ -1053,6 +1054,8 @@ int main(int argc, char * * argv) return true; }); + initPlugins(); + if (!op) throw UsageError("no operation specified"); if (op != opDump && op != opRestore) /* !!! hack */ diff --git a/src/nix/build.cc b/src/nix/build.cc index f7c99f12dbbf..b329ac38ac2b 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -50,7 +50,9 @@ struct CmdBuild : MixDryRun, InstallablesCommand void run(ref<Store> store) override { - auto buildables = toBuildables(store, dryRun ? DryRun : Build, installables); + auto buildables = build(store, dryRun ? DryRun : Build, installables); + + if (dryRun) return; for (size_t i = 0; i < buildables.size(); ++i) { auto & b(buildables[i]); diff --git a/src/nix/command.hh b/src/nix/command.hh index 6b34e3881e79..97a6fee7fd27 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -5,6 +5,8 @@ namespace nix { +extern std::string programPath; + struct Value; class Bindings; class EvalState; @@ -196,7 +198,7 @@ std::shared_ptr<Installable> parseInstallable( SourceExprCommand & cmd, ref<Store> store, const std::string & installable, bool useDefaultInstallables); -Buildables toBuildables(ref<Store> store, RealiseMode mode, +Buildables build(ref<Store> store, RealiseMode mode, std::vector<std::shared_ptr<Installable>> installables); PathSet toStorePaths(ref<Store> store, RealiseMode mode, diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 2ddea9e70a6a..f29429c1ac49 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -57,15 +57,15 @@ struct CmdCopy : StorePathsCommand return { Example{ "To copy Firefox from the local store to a binary cache in file:///tmp/cache:", - "nix copy --to file:///tmp/cache -r $(type -p firefox)" + "nix copy --to file:///tmp/cache $(type -p firefox)" }, Example{ "To copy the entire current NixOS system closure to another machine via SSH:", - "nix copy --to ssh://server -r /run/current-system" + "nix copy --to ssh://server /run/current-system" }, Example{ "To copy a closure from another machine via SSH:", - "nix copy --from ssh://server -r /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2" + "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2" }, }; } diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 1a1866437b07..f411c0cb7c89 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -29,6 +29,7 @@ struct CmdDumpPath : StorePathCommand { FdSink sink(STDOUT_FILENO); store->narFromPath(storePath, sink); + sink.flush(); } }; diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 127be321eee2..7eaa86e2f914 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -52,7 +52,12 @@ struct CmdEdit : InstallableCommand throw Error("cannot parse meta.position attribute '%s'", pos); std::string filename(pos, 0, colon); - int lineno = std::stoi(std::string(pos, colon + 1)); + int lineno; + try { + lineno = std::stoi(std::string(pos, colon + 1)); + } catch (std::invalid_argument e) { + throw Error("cannot parse line number '%s'", pos); + } auto editor = getEnv("EDITOR", "cat"); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 0fbeca1c121d..b7058361cbec 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -5,10 +5,11 @@ #include "eval.hh" #include "json.hh" #include "value-to-json.hh" +#include "progress-bar.hh" using namespace nix; -struct CmdEval : MixJSON, InstallablesCommand +struct CmdEval : MixJSON, InstallableCommand { bool raw = false; @@ -56,20 +57,19 @@ struct CmdEval : MixJSON, InstallablesCommand auto state = getEvalState(); - auto jsonOut = json ? std::make_unique<JSONList>(std::cout) : nullptr; + auto v = installable->toValue(*state); + PathSet context; - for (auto & i : installables) { - auto v = i->toValue(*state); - PathSet context; - if (raw) { - std::cout << state->coerceToString(noPos, *v, context); - } else if (json) { - auto jsonElem = jsonOut->placeholder(); - printValueAsJSON(*state, true, *v, jsonElem, context); - } else { - state->forceValueDeep(*v); - std::cout << *v << "\n"; - } + stopProgressBar(); + + if (raw) { + std::cout << state->coerceToString(noPos, *v, context); + } else if (json) { + JSONPlaceholder jsonOut(std::cout); + printValueAsJSON(*state, true, *v, jsonOut, context); + } else { + state->forceValueDeep(*v); + std::cout << *v << "\n"; } } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index ae93c4ef649e..a3fdd8a2808d 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -30,10 +30,8 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) vSourceExpr = state.allocValue(); - if (file != "") { - Expr * e = state.parseExprFromFile(resolveExprPath(lookupFileArg(state, file))); - state.eval(e, *vSourceExpr); - } + if (file != "") + state.evalFile(lookupFileArg(state, file), *vSourceExpr); else { @@ -255,7 +253,7 @@ std::shared_ptr<Installable> parseInstallable( return installables.front(); } -Buildables toBuildables(ref<Store> store, RealiseMode mode, +Buildables build(ref<Store> store, RealiseMode mode, std::vector<std::shared_ptr<Installable>> installables) { if (mode != Build) @@ -293,7 +291,7 @@ PathSet toStorePaths(ref<Store> store, RealiseMode mode, { PathSet outPaths; - for (auto & b : toBuildables(store, mode, installables)) + for (auto & b : build(store, mode, installables)) for (auto & output : b.outputs) outPaths.insert(output.second); diff --git a/src/nix/local.mk b/src/nix/local.mk index bddd53b168d3..f76da194467c 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -6,4 +6,6 @@ nix_SOURCES := $(wildcard $(d)/*.cc) $(wildcard src/linenoise/*.cpp) nix_LIBS = libexpr libmain libstore libutil libformat +nix_LDFLAGS = -pthread + $(eval $(call install-symlink, nix, $(bindir)/nix-hash)) diff --git a/src/nix/log.cc b/src/nix/log.cc index 966ad8b65087..f07ec4e93a16 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -50,6 +50,7 @@ struct CmdLog : InstallableCommand auto b = installable->toBuildable(); + RunPager pager; for (auto & sub : subs) { auto log = b.drvPath != "" ? sub->getBuildLog(b.drvPath) : nullptr; for (auto & output : b.outputs) { diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 69620595d8ca..e99622faf472 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -90,6 +90,16 @@ struct CmdLsStore : StoreCommand, MixLs expectArg("path", &path); } + Examples examples() override + { + return { + Example{ + "To list the contents of a store path in a binary cache:", + "nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10" + }, + }; + } + std::string name() override { return "ls-store"; @@ -116,6 +126,16 @@ struct CmdLsNar : Command, MixLs expectArg("path", &path); } + Examples examples() override + { + return { + Example{ + "To list a specific file in a NAR:", + "nix ls-nar -l hello.nar /bin/hello" + }, + }; + } + std::string name() override { return "ls-nar"; diff --git a/src/nix/main.cc b/src/nix/main.cc index 06bb8a1c3043..bb107ec7d3f6 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -16,6 +16,8 @@ void chrootHelper(int argc, char * * argv); namespace nix { +std::string programPath; + struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") @@ -78,7 +80,8 @@ void mainWrapped(int argc, char * * argv) initNix(); initGC(); - string programName = baseNameOf(argv[0]); + programPath = argv[0]; + string programName = baseNameOf(programPath); { auto legacy = (*RegisterLegacyCommand::commands)[programName]; @@ -89,6 +92,8 @@ void mainWrapped(int argc, char * * argv) args.parseCmdline(argvToStrings(argc, argv)); + initPlugins(); + if (!args.command) args.showHelpAndExit(); Finally f([]() { stopProgressBar(); }); diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc new file mode 100644 index 000000000000..310942574a2a --- /dev/null +++ b/src/nix/ping-store.cc @@ -0,0 +1,35 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdPingStore : StoreCommand +{ + std::string name() override + { + return "ping-store"; + } + + std::string description() override + { + return "test whether a store can be opened"; + } + + Examples examples() override + { + return { + Example{ + "To test whether connecting to a remote Nix store via SSH works:", + "nix ping-store --store ssh://mac1" + }, + }; + } + + void run(ref<Store> store) override + { + store->connect(); + } +}; + +static RegisterCommand r1(make_ref<CmdPingStore>()); diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index fb9955190b40..e6553c06f4ae 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -3,8 +3,9 @@ #include "sync.hh" #include "store-api.hh" -#include <map> #include <atomic> +#include <map> +#include <thread> namespace nix { @@ -22,44 +23,6 @@ static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n) return fields[n].i; } -/* Truncate a string to 'width' printable characters. ANSI escape - sequences are copied but not included in the character count. Also, - tabs are expanded to spaces. */ -static std::string ansiTruncate(const std::string & s, int width) -{ - if (width <= 0) return s; - - std::string t; - size_t w = 0; - auto i = s.begin(); - - while (w < (size_t) width && i != s.end()) { - if (*i == '\e') { - t += *i++; - if (i != s.end() && *i == '[') { - t += *i++; - while (i != s.end() && (*i < 0x40 || *i > 0x7e)) { - t += *i++; - } - if (i != s.end()) t += *i++; - } - } - - else if (*i == '\t') { - t += ' '; w++; - while (w < (size_t) width && w & 8) { - t += ' '; w++; - } - } - - else { - t += *i++; w++; - } - } - - return t; -} - class ProgressBar : public Logger { private: @@ -101,15 +64,28 @@ private: Sync<State> state_; + std::thread updateThread; + + std::condition_variable quitCV, updateCV; + public: ProgressBar() { + updateThread = std::thread([&]() { + auto state(state_.lock()); + while (state->active) { + state.wait(updateCV); + draw(*state); + state.wait_for(quitCV, std::chrono::milliseconds(50)); + } + }); } ~ProgressBar() { stop(); + updateThread.join(); } void stop() @@ -121,6 +97,8 @@ public: writeToStderr("\r\e[K"); if (status != "") writeToStderr("[" + status + "]\n"); + updateCV.notify_one(); + quitCV.notify_one(); } void log(Verbosity lvl, const FormatOrString & fs) override @@ -132,7 +110,7 @@ public: void log(State & state, Verbosity lvl, const std::string & s) { writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n"); - update(state); + draw(state); } void startActivity(ActivityId act, Verbosity lvl, ActivityType type, @@ -167,7 +145,12 @@ public: if (type == actSubstitute) { auto name = storePathToName(getS(fields, 0)); - i->s = fmt("fetching " ANSI_BOLD "%s" ANSI_NORMAL " from %s", name, getS(fields, 1)); + auto sub = getS(fields, 1); + i->s = fmt( + hasPrefix(sub, "local") + ? "copying " ANSI_BOLD "%s" ANSI_NORMAL " from %s" + : "fetching " ANSI_BOLD "%s" ANSI_NORMAL " from %s", + name, sub); } if (type == actQueryPathInfo) { @@ -180,7 +163,7 @@ public: || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) i->visible = false; - update(*state); + update(); } /* Check whether an activity has an ancestore with the specified @@ -215,7 +198,7 @@ public: state->its.erase(i); } - update(*state); + update(); } void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override @@ -225,7 +208,7 @@ public: if (type == resFileLinked) { state->filesLinked++; state->bytesLinked += getI(fields, 0); - update(*state); + update(); } else if (type == resBuildLogLine) { @@ -238,25 +221,25 @@ public: info.lastLine = lastLine; state->activities.emplace_back(info); i->second = std::prev(state->activities.end()); - update(*state); + update(); } } else if (type == resUntrustedPath) { state->untrustedPaths++; - update(*state); + update(); } else if (type == resCorruptedPath) { state->corruptedPaths++; - update(*state); + update(); } else if (type == resSetPhase) { auto i = state->its.find(act); assert(i != state->its.end()); i->second->phase = getS(fields, 0); - update(*state); + update(); } else if (type == resProgress) { @@ -267,7 +250,7 @@ public: actInfo.expected = getI(fields, 1); actInfo.running = getI(fields, 2); actInfo.failed = getI(fields, 3); - update(*state); + update(); } else if (type == resSetExpected) { @@ -279,17 +262,16 @@ public: state->activitiesByType[type].expected -= j; j = getI(fields, 1); state->activitiesByType[type].expected += j; - update(*state); + update(); } } void update() { - auto state(state_.lock()); - update(*state); + updateCV.notify_one(); } - void update(State & state) + void draw(State & state) { if (!state.active) return; @@ -323,7 +305,10 @@ public: } } - writeToStderr("\r" + ansiTruncate(line, getWindowSize().second) + "\e[K"); + auto width = getWindowSize().second; + if (width <= 0) std::numeric_limits<decltype(width)>::max(); + + writeToStderr("\r" + filterANSIEscapes(line, width) + "\e[K"); } std::string getStatus(State & state) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 1adb816c5bf0..9216209173d9 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -186,7 +186,16 @@ bool NixRepl::getLine(string & input, const std::string &prompt) { char * s = linenoise(prompt.c_str()); Finally doFree([&]() { free(s); }); - if (!s) return false; + if (!s) { + switch (auto type = linenoiseKeyType()) { + case 1: // ctrl-C + return true; + case 2: // ctrl-D + return false; + default: + throw Error(format("Unexpected linenoise keytype: %1%") % type); + } + } input += s; return true; } diff --git a/src/nix/run.cc b/src/nix/run.cc index ade87e63a49c..d04e106e037b 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -16,8 +16,6 @@ using namespace nix; std::string chrootHelperName = "__run_in_chroot"; -extern char * * environ; - struct CmdRun : InstallablesCommand { std::vector<std::string> command = { "bash" }; @@ -85,6 +83,10 @@ struct CmdRun : InstallablesCommand "To run GNU Hello:", "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'" }, + Example{ + "To run GNU Hello in a chroot store:", + "nix run --store ~/my-nix nixpkgs.hello -c hello" + }, }; } @@ -105,7 +107,7 @@ struct CmdRun : InstallablesCommand if (s) kept[var] = s; } - environ = nullptr; + clearEnv(); for (auto & var : kept) setenv(var.first.c_str(), var.second.c_str(), 1); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc new file mode 100644 index 000000000000..758bbbc688bc --- /dev/null +++ b/src/nix/upgrade-nix.cc @@ -0,0 +1,131 @@ +#include "command.hh" +#include "store-api.hh" +#include "download.hh" +#include "eval.hh" +#include "attr-path.hh" + +using namespace nix; + +struct CmdUpgradeNix : StoreCommand +{ + Path profileDir; + + CmdUpgradeNix() + { + mkFlag() + .longName("profile") + .shortName('p') + .labels({"profile-dir"}) + .description("the Nix profile to upgrade") + .dest(&profileDir); + } + + std::string name() override + { + return "upgrade-nix"; + } + + std::string description() override + { + return "upgrade Nix to the latest stable version"; + } + + Examples examples() override + { + return { + Example{ + "To upgrade Nix to the latest stable version:", + "nix upgrade-nix" + }, + Example{ + "To upgrade Nix in a specific profile:", + "nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile" + }, + }; + } + + void run(ref<Store> store) override + { + settings.pureEval = true; + + if (profileDir == "") + profileDir = getProfileDir(store); + + printInfo("upgrading Nix in profile '%s'", profileDir); + + Path storePath; + { + Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); + storePath = getLatestNix(store); + } + + { + Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", storePath)); + store->ensurePath(storePath); + } + + { + Activity act(*logger, lvlInfo, actUnknown, fmt("verifying that '%s' works...", storePath)); + auto program = storePath + "/bin/nix-env"; + auto s = runProgram(program, false, {"--version"}); + if (s.find("Nix") == std::string::npos) + throw Error("could not verify that '%s' works", program); + } + + { + Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", storePath, profileDir)); + runProgram(settings.nixBinDir + "/nix-env", false, + {"--profile", profileDir, "-i", storePath, "--no-sandbox"}); + } + } + + /* Return the profile in which Nix is installed. */ + Path getProfileDir(ref<Store> store) + { + Path where; + + for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":")) + if (pathExists(dir + "/nix-env")) { + where = dir; + break; + } + + if (where == "") + throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); + + printInfo("found Nix in '%s'", where); + + if (hasPrefix(where, "/run/current-system")) + throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); + + Path profileDir; + Path userEnv; + + if (baseNameOf(where) != "bin" || + !hasSuffix(userEnv = canonPath(profileDir = dirOf(where), true), "user-environment")) + throw Error("directory '%s' does not appear to be part of a Nix profile", where); + + if (!store->isValidPath(userEnv)) + throw Error("directory '%s' is not in the Nix store", userEnv); + + return profileDir; + } + + /* Return the store path of the latest stable Nix. */ + Path getLatestNix(ref<Store> store) + { + // FIXME: use nixos.org? + auto req = DownloadRequest("https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"); + auto res = getDownloader()->download(req); + + EvalState state(Strings(), store); + auto v = state.allocValue(); + state.eval(state.parseExprFromString(*res.data, "/no-such-path"), *v); + Bindings & bindings(*state.allocBindings(0)); + auto v2 = findAlongAttrPath(state, settings.thisSystem, bindings, *v); + + return state.forceString(*v2); + } +}; + +static RegisterCommand r1(make_ref<CmdUpgradeNix>()); diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp index 9754e464c76b..5b0b0ea5b301 100644 --- a/src/nlohmann/json.hpp +++ b/src/nlohmann/json.hpp @@ -1,7 +1,7 @@ /* __ _____ _____ _____ __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 2.1.1 +| | |__ | | | | | | version 3.0.1 |_____|_____|_____|_|___| https://github.com/nlohmann/json Licensed under the MIT License <http://opensource.org/licenses/MIT>. @@ -109,7 +109,7 @@ SOFTWARE. #define JSON_UNLIKELY(x) x #endif -// cpp language standard detection +// C++ language standard detection #if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 #define JSON_HAS_CPP_17 #define JSON_HAS_CPP_14 @@ -128,20 +128,18 @@ template<typename = void, typename = void> struct adl_serializer; // forward declaration of basic_json (required to split the class) -template<template<typename U, typename V, typename... Args> class ObjectType = - std::map, - template<typename U, typename... Args> class ArrayType = std::vector, +template<template<typename, typename, typename...> class ObjectType = std::map, + template<typename, typename...> class ArrayType = std::vector, class StringType = std::string, class BooleanType = bool, class NumberIntegerType = std::int64_t, class NumberUnsignedType = std::uint64_t, class NumberFloatType = double, - template<typename U> class AllocatorType = std::allocator, - template<typename T, typename SFINAE = void> class JSONSerializer = - adl_serializer> + template<typename> class AllocatorType = std::allocator, + template<typename, typename = void> class JSONSerializer = adl_serializer> class basic_json; -// Ugly macros to avoid uglier copy-paste when specializing basic_json -// This is only temporary and will be removed in 3.0 +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. #define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ template<template<typename, typename, typename...> class ObjectType, \ @@ -227,7 +225,7 @@ class exception : public std::exception /*! @brief exception indicating a parse error -This excpetion is thrown by the library when a parse error occurs. Parse errors +This exception is thrown by the library when a parse error occurs. Parse errors can occur during the deserialization of JSON text, CBOR, MessagePack, as well as when using JSON Patch. @@ -243,12 +241,12 @@ json.exception.parse_error.102 | parse error at 14: missing or wrong low surroga json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid. json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects. json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors. -json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number wihtout a leading `0`. +json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`. json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character. json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences. json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number. json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. -json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xf8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. +json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. @note For an input with n bytes, 1 is the index of the first character and n+1 @@ -378,6 +376,7 @@ json.exception.type_error.312 | cannot use update() with string | The @ref updat json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined. json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. +json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. | @liveexample{The following code shows how a `type_error` exception can be caught.,type_error} @@ -457,7 +456,6 @@ Exceptions have ids 5xx. name / id | example message | description ------------------------------ | --------------- | ------------------------- json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed. -json.exception.other_error.502 | invalid object size for conversion | Some conversions to user-defined types impose constraints on the object size (e.g. std::pair) @sa @ref exception for the base class of the library exceptions @sa @ref parse_error for exceptions indicating a parse error @@ -540,20 +538,14 @@ Returns an ordering that is similar to Python: inline bool operator<(const value_t lhs, const value_t rhs) noexcept { static constexpr std::array<uint8_t, 8> order = {{ - 0, // null - 3, // object - 4, // array - 5, // string - 1, // boolean - 2, // integer - 2, // unsigned - 2, // float + 0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */, + 1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */ } }; const auto l_index = static_cast<std::size_t>(lhs); const auto r_index = static_cast<std::size_t>(rhs); - return (l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index]); + return l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index]; } @@ -591,17 +583,15 @@ struct merge_and_renumber; template<std::size_t... I1, std::size_t... I2> struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>> - : index_sequence < I1..., (sizeof...(I1) + I2)... > - {}; + : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; template<std::size_t N> struct make_index_sequence : merge_and_renumber < typename make_index_sequence < N / 2 >::type, - typename make_index_sequence < N - N / 2 >::type > -{}; + typename make_index_sequence < N - N / 2 >::type > {}; -template<> struct make_index_sequence<0> : index_sequence<> { }; -template<> struct make_index_sequence<1> : index_sequence<0> { }; +template<> struct make_index_sequence<0> : index_sequence<> {}; +template<> struct make_index_sequence<1> : index_sequence<0> {}; template<typename... Ts> using index_sequence_for = make_index_sequence<sizeof...(Ts)>; @@ -624,7 +614,7 @@ template<class B1> struct conjunction<B1> : B1 {}; template<class B1, class... Bn> struct conjunction<B1, Bn...> : std::conditional<bool(B1::value), conjunction<Bn...>, B1>::type {}; -template<class B> struct negation : std::integral_constant < bool, !B::value > {}; +template<class B> struct negation : std::integral_constant<bool, not B::value> {}; // dispatch utility (taken from ranges-v3) template<unsigned N> struct priority_tag : priority_tag < N - 1 > {}; @@ -725,8 +715,7 @@ struct external_constructor<value_t::array> } template<typename BasicJsonType, typename CompatibleArrayType, - enable_if_t<not std::is_same<CompatibleArrayType, - typename BasicJsonType::array_t>::value, + enable_if_t<not std::is_same<CompatibleArrayType, typename BasicJsonType::array_t>::value, int> = 0> static void construct(BasicJsonType& j, const CompatibleArrayType& arr) { @@ -743,7 +732,7 @@ struct external_constructor<value_t::array> j.m_type = value_t::array; j.m_value = value_t::array; j.m_value.array->reserve(arr.size()); - for (bool x : arr) + for (const bool x : arr) { j.m_value.array->push_back(x); } @@ -782,8 +771,7 @@ struct external_constructor<value_t::object> } template<typename BasicJsonType, typename CompatibleObjectType, - enable_if_t<not std::is_same<CompatibleObjectType, - typename BasicJsonType::object_t>::value, int> = 0> + enable_if_t<not std::is_same<CompatibleObjectType, typename BasicJsonType::object_t>::value, int> = 0> static void construct(BasicJsonType& j, const CompatibleObjectType& obj) { using std::begin; @@ -896,7 +884,7 @@ struct is_compatible_integer_type is_compatible_integer_type_impl < std::is_integral<CompatibleNumberIntegerType>::value and not std::is_same<bool, CompatibleNumberIntegerType>::value, - RealIntegerType, CompatibleNumberIntegerType > ::value; + RealIntegerType, CompatibleNumberIntegerType >::value; }; @@ -922,10 +910,8 @@ template<typename BasicJsonType, typename T> struct has_non_default_from_json { private: - template < - typename U, - typename = enable_if_t<std::is_same< - T, decltype(uncvref_t<U>::from_json(std::declval<BasicJsonType>()))>::value >> + template<typename U, typename = + enable_if_t<std::is_same<T, decltype(uncvref_t<U>::from_json(std::declval<BasicJsonType>()))>::value>> static int detect(U&&); static void detect(...); @@ -954,22 +940,21 @@ struct has_to_json // to_json // ///////////// -template<typename BasicJsonType, typename T, enable_if_t< - std::is_same<T, typename BasicJsonType::boolean_t>::value, int> = 0> +template<typename BasicJsonType, typename T, + enable_if_t<std::is_same<T, typename BasicJsonType::boolean_t>::value, int> = 0> void to_json(BasicJsonType& j, T b) noexcept { external_constructor<value_t::boolean>::construct(j, b); } template<typename BasicJsonType, typename CompatibleString, - enable_if_t<std::is_constructible<typename BasicJsonType::string_t, - CompatibleString>::value, int> = 0> + enable_if_t<std::is_constructible<typename BasicJsonType::string_t, CompatibleString>::value, int> = 0> void to_json(BasicJsonType& j, const CompatibleString& s) { external_constructor<value_t::string>::construct(j, s); } -template <typename BasicJsonType> +template<typename BasicJsonType> void to_json(BasicJsonType& j, typename BasicJsonType::string_t&& s) { external_constructor<value_t::string>::construct(j, std::move(s)); @@ -982,19 +967,15 @@ void to_json(BasicJsonType& j, FloatType val) noexcept external_constructor<value_t::number_float>::construct(j, static_cast<typename BasicJsonType::number_float_t>(val)); } -template < - typename BasicJsonType, typename CompatibleNumberUnsignedType, - enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_unsigned_t, - CompatibleNumberUnsignedType>::value, int> = 0 > +template<typename BasicJsonType, typename CompatibleNumberUnsignedType, + enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_unsigned_t, CompatibleNumberUnsignedType>::value, int> = 0> void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept { external_constructor<value_t::number_unsigned>::construct(j, static_cast<typename BasicJsonType::number_unsigned_t>(val)); } -template < - typename BasicJsonType, typename CompatibleNumberIntegerType, - enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_integer_t, - CompatibleNumberIntegerType>::value, int> = 0 > +template<typename BasicJsonType, typename CompatibleNumberIntegerType, + enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_integer_t, CompatibleNumberIntegerType>::value, int> = 0> void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept { external_constructor<value_t::number_integer>::construct(j, static_cast<typename BasicJsonType::number_integer_t>(val)); @@ -1014,49 +995,43 @@ void to_json(BasicJsonType& j, const std::vector<bool>& e) external_constructor<value_t::array>::construct(j, e); } -template < - typename BasicJsonType, typename CompatibleArrayType, - enable_if_t < - is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value or - std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value, - int > = 0 > +template<typename BasicJsonType, typename CompatibleArrayType, + enable_if_t<is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value or + std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value, + int> = 0> void to_json(BasicJsonType& j, const CompatibleArrayType& arr) { external_constructor<value_t::array>::construct(j, arr); } -template <typename BasicJsonType, typename T, - enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0> +template<typename BasicJsonType, typename T, + enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0> void to_json(BasicJsonType& j, std::valarray<T> arr) { external_constructor<value_t::array>::construct(j, std::move(arr)); } -template <typename BasicJsonType> +template<typename BasicJsonType> void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr) { external_constructor<value_t::array>::construct(j, std::move(arr)); } -template < - typename BasicJsonType, typename CompatibleObjectType, - enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, - int> = 0 > +template<typename BasicJsonType, typename CompatibleObjectType, + enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0> void to_json(BasicJsonType& j, const CompatibleObjectType& obj) { external_constructor<value_t::object>::construct(j, obj); } -template <typename BasicJsonType> +template<typename BasicJsonType> void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj) { external_constructor<value_t::object>::construct(j, std::move(obj)); } template<typename BasicJsonType, typename T, std::size_t N, - enable_if_t<not std::is_constructible< - typename BasicJsonType::string_t, T (&)[N]>::value, - int> = 0> + enable_if_t<not std::is_constructible<typename BasicJsonType::string_t, T (&)[N]>::value, int> = 0> void to_json(BasicJsonType& j, T (&arr)[N]) { external_constructor<value_t::array>::construct(j, arr); @@ -1087,8 +1062,7 @@ void to_json(BasicJsonType& j, const std::tuple<Args...>& t) // overloads for basic_json template parameters template<typename BasicJsonType, typename ArithmeticType, enable_if_t<std::is_arithmetic<ArithmeticType>::value and - not std::is_same<ArithmeticType, - typename BasicJsonType::boolean_t>::value, + not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, int> = 0> void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) { @@ -1351,6 +1325,13 @@ struct to_json_fn { static_assert(sizeof(BasicJsonType) == 0, "could not find to_json() method in T's namespace"); + +#ifdef _MSC_VER + // MSVC does not show a stacktrace for the above assert + using decayed = uncvref_t<T>; + static_assert(sizeof(typename decayed::force_msvc_stacktrace) == 0, + "forcing MSVC stacktrace to show which T we're talking about."); +#endif } public: @@ -1378,6 +1359,12 @@ struct from_json_fn { static_assert(sizeof(BasicJsonType) == 0, "could not find from_json() method in T's namespace"); +#ifdef _MSC_VER + // MSVC does not show a stacktrace for the above assert + using decayed = uncvref_t<T>; + static_assert(sizeof(typename decayed::force_msvc_stacktrace) == 0, + "forcing MSVC stacktrace to show which T we're talking about."); +#endif } public: @@ -1448,7 +1435,7 @@ class input_stream_adapter : public input_adapter_protocol explicit input_stream_adapter(std::istream& i) : is(i), sb(*i.rdbuf()) { - // ignore Byte Order Mark at start of input + // skip byte order mark std::char_traits<char>::int_type c; if ((c = get_character()) == 0xEF) { @@ -1472,7 +1459,7 @@ class input_stream_adapter : public input_adapter_protocol } else if (c != std::char_traits<char>::eof()) { - is.unget(); // Not BOM. Process as usual. + is.unget(); // no byte order mark; process as usual } } @@ -1481,8 +1468,8 @@ class input_stream_adapter : public input_adapter_protocol input_stream_adapter& operator=(input_stream_adapter&) = delete; // std::istream/std::streambuf use std::char_traits<char>::to_int_type, to - // ensure that std::char_traits<char>::eof() and the character 0xff do not - // end up as the same value, eg. 0xffffffff. + // ensure that std::char_traits<char>::eof() and the character 0xFF do not + // end up as the same value, eg. 0xFFFFFFFF. std::char_traits<char>::int_type get_character() override { return sb.sbumpc(); @@ -1561,8 +1548,7 @@ class input_adapter template<typename CharT, typename std::enable_if< std::is_pointer<CharT>::value and - std::is_integral< - typename std::remove_pointer<CharT>::type>::value and + std::is_integral<typename std::remove_pointer<CharT>::type>::value and sizeof(typename std::remove_pointer<CharT>::type) == 1, int>::type = 0> input_adapter(CharT b, std::size_t l) @@ -1574,8 +1560,7 @@ class input_adapter template<typename CharT, typename std::enable_if< std::is_pointer<CharT>::value and - std::is_integral< - typename std::remove_pointer<CharT>::type>::value and + std::is_integral<typename std::remove_pointer<CharT>::type>::value and sizeof(typename std::remove_pointer<CharT>::type) == 1, int>::type = 0> input_adapter(CharT b) @@ -1585,8 +1570,7 @@ class input_adapter /// input adapter for iterator range with contiguous storage template<class IteratorType, typename std::enable_if< - std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, - std::random_access_iterator_tag>::value, + std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value, int>::type = 0> input_adapter(IteratorType first, IteratorType last) { @@ -1624,13 +1608,10 @@ class input_adapter : input_adapter(std::begin(array), std::end(array)) {} /// input adapter for contiguous container - template < - class ContiguousContainer, - typename std::enable_if < - not std::is_pointer<ContiguousContainer>::value and - std::is_base_of<std::random_access_iterator_tag, - typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value, - int >::type = 0 > + template<class ContiguousContainer, typename + std::enable_if<not std::is_pointer<ContiguousContainer>::value and + std::is_base_of<std::random_access_iterator_tag, typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value, + int>::type = 0> input_adapter(const ContiguousContainer& c) : input_adapter(std::begin(c), std::end(c)) {} @@ -1804,6 +1785,12 @@ class lexer checks if it is inside the range. If a violation was detected, set up an error message and return false. Otherwise, return true. + @param[in] ranges list of integers; interpreted as list of pairs of + inclusive lower and upper bound, respectively + + @pre The passed list @a ranges must have 2, 4, or 6 elements; that is, + 1, 2, or 3 pairs. This precondition is enforced by an assertion. + @return true if and only if no range violation was detected */ bool next_byte_in_range(std::initializer_list<int> ranges) @@ -1970,19 +1957,19 @@ class lexer // result of the above calculation yields a proper codepoint assert(0x00 <= codepoint and codepoint <= 0x10FFFF); - // translate code point to bytes + // translate codepoint into bytes if (codepoint < 0x80) { // 1-byte characters: 0xxxxxxx (ASCII) add(codepoint); } - else if (codepoint <= 0x7ff) + else if (codepoint <= 0x7FF) { // 2-byte characters: 110xxxxx 10xxxxxx add(0xC0 | (codepoint >> 6)); add(0x80 | (codepoint & 0x3F)); } - else if (codepoint <= 0xffff) + else if (codepoint <= 0xFFFF) { // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx add(0xE0 | (codepoint >> 12)); @@ -2021,12 +2008,12 @@ class lexer case 0x07: case 0x08: case 0x09: - case 0x0a: - case 0x0b: - case 0x0c: - case 0x0d: - case 0x0e: - case 0x0f: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: case 0x10: case 0x11: case 0x12: @@ -2037,12 +2024,12 @@ class lexer case 0x17: case 0x18: case 0x19: - case 0x1a: - case 0x1b: - case 0x1c: - case 0x1d: - case 0x1e: - case 0x1f: + case 0x1A: + case 0x1B: + case 0x1C: + case 0x1D: + case 0x1E: + case 0x1F: { error_message = "invalid string: control character must be escaped"; return token_type::parse_error; @@ -2058,12 +2045,12 @@ class lexer case 0x27: case 0x28: case 0x29: - case 0x2a: - case 0x2b: - case 0x2c: - case 0x2d: - case 0x2e: - case 0x2f: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: case 0x30: case 0x31: case 0x32: @@ -2074,12 +2061,12 @@ class lexer case 0x37: case 0x38: case 0x39: - case 0x3a: - case 0x3b: - case 0x3c: - case 0x3d: - case 0x3e: - case 0x3f: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: case 0x40: case 0x41: case 0x42: @@ -2090,12 +2077,12 @@ class lexer case 0x47: case 0x48: case 0x49: - case 0x4a: - case 0x4b: - case 0x4c: - case 0x4d: - case 0x4e: - case 0x4f: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: case 0x50: case 0x51: case 0x52: @@ -2106,11 +2093,11 @@ class lexer case 0x57: case 0x58: case 0x59: - case 0x5a: - case 0x5b: - case 0x5d: - case 0x5e: - case 0x5f: + case 0x5A: + case 0x5B: + case 0x5D: + case 0x5E: + case 0x5F: case 0x60: case 0x61: case 0x62: @@ -2121,12 +2108,12 @@ class lexer case 0x67: case 0x68: case 0x69: - case 0x6a: - case 0x6b: - case 0x6c: - case 0x6d: - case 0x6e: - case 0x6f: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: case 0x70: case 0x71: case 0x72: @@ -2137,48 +2124,48 @@ class lexer case 0x77: case 0x78: case 0x79: - case 0x7a: - case 0x7b: - case 0x7c: - case 0x7d: - case 0x7e: - case 0x7f: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: { add(current); break; } // U+0080..U+07FF: bytes C2..DF 80..BF - case 0xc2: - case 0xc3: - case 0xc4: - case 0xc5: - case 0xc6: - case 0xc7: - case 0xc8: - case 0xc9: - case 0xca: - case 0xcb: - case 0xcc: - case 0xcd: - case 0xce: - case 0xcf: - case 0xd0: - case 0xd1: - case 0xd2: - case 0xd3: - case 0xd4: - case 0xd5: - case 0xd6: - case 0xd7: - case 0xd8: - case 0xd9: - case 0xda: - case 0xdb: - case 0xdc: - case 0xdd: - case 0xde: - case 0xdf: + case 0xC2: + case 0xC3: + case 0xC4: + case 0xC5: + case 0xC6: + case 0xC7: + case 0xC8: + case 0xC9: + case 0xCA: + case 0xCB: + case 0xCC: + case 0xCD: + case 0xCE: + case 0xCF: + case 0xD0: + case 0xD1: + case 0xD2: + case 0xD3: + case 0xD4: + case 0xD5: + case 0xD6: + case 0xD7: + case 0xD8: + case 0xD9: + case 0xDA: + case 0xDB: + case 0xDC: + case 0xDD: + case 0xDE: + case 0xDF: { if (JSON_UNLIKELY(not next_byte_in_range({0x80, 0xBF}))) { @@ -2188,7 +2175,7 @@ class lexer } // U+0800..U+0FFF: bytes E0 A0..BF 80..BF - case 0xe0: + case 0xE0: { if (JSON_UNLIKELY(not (next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF})))) { @@ -2199,20 +2186,20 @@ class lexer // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF - case 0xe1: - case 0xe2: - case 0xe3: - case 0xe4: - case 0xe5: - case 0xe6: - case 0xe7: - case 0xe8: - case 0xe9: - case 0xea: - case 0xeb: - case 0xec: - case 0xee: - case 0xef: + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xEE: + case 0xEF: { if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF})))) { @@ -2222,7 +2209,7 @@ class lexer } // U+D000..U+D7FF: bytes ED 80..9F 80..BF - case 0xed: + case 0xED: { if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x9F, 0x80, 0xBF})))) { @@ -2232,7 +2219,7 @@ class lexer } // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF - case 0xf0: + case 0xF0: { if (JSON_UNLIKELY(not (next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) { @@ -2242,9 +2229,9 @@ class lexer } // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF - case 0xf1: - case 0xf2: - case 0xf3: + case 0xF1: + case 0xF2: + case 0xF3: { if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) { @@ -2254,7 +2241,7 @@ class lexer } // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF - case 0xf4: + case 0xF4: { if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) { @@ -2772,9 +2759,9 @@ scan_number_done: { // escape control characters std::string result; - for (auto c : token_string) + for (const auto c : token_string) { - if ('\x00' <= c and c <= '\x1f') + if ('\x00' <= c and c <= '\x1F') { // escape control characters std::stringstream ss; @@ -2877,10 +2864,10 @@ scan_number_done: std::size_t chars_read = 0; /// raw input token string (for error messages) - std::vector<char> token_string { }; + std::vector<char> token_string {}; /// buffer for variable-length tokens (numbers, strings) - std::string yytext { }; + std::string yytext {}; /// a description of occurred lexer errors const char* error_message = ""; @@ -3281,7 +3268,7 @@ class parser } /*! - @brief the acutal acceptor + @brief the actual acceptor @invariant 1. The last token is not yet processed. Therefore, the caller of this function must make sure a token has been read. @@ -3539,7 +3526,7 @@ class primitive_iterator_t return *this; } - primitive_iterator_t operator++(int) + primitive_iterator_t const operator++(int) { auto result = *this; m_it++; @@ -3552,7 +3539,7 @@ class primitive_iterator_t return *this; } - primitive_iterator_t operator--(int) + primitive_iterator_t const operator--(int) { auto result = *this; m_it--; @@ -3618,7 +3605,7 @@ This class implements a both iterators (iterator and const_iterator) for the iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593) */ template<typename BasicJsonType> -class iter_impl : public std::iterator<std::bidirectional_iterator_tag, BasicJsonType> +class iter_impl { /// allow basic_json to access private members friend iter_impl<typename std::conditional<std::is_const<BasicJsonType>::value, typename std::remove_const<BasicJsonType>::type, const BasicJsonType>::type>; @@ -3632,6 +3619,14 @@ class iter_impl : public std::iterator<std::bidirectional_iterator_tag, BasicJso "iter_impl only accepts (const) basic_json"); public: + + /// The std::iterator class template (used as a base class to provide typedefs) is deprecated in C++17. + /// The C++ Standard has never required user-defined iterators to derive from std::iterator. + /// A user-defined iterator should provide publicly accessible typedefs named + /// iterator_category, value_type, difference_type, pointer, and reference. + /// Note that value_type is required to be non-const, even for constant iterators. + using iterator_category = std::bidirectional_iterator_tag; + /// the type of the values when the iterator is dereferenced using value_type = typename BasicJsonType::value_type; /// a type to represent differences between iterators @@ -3855,7 +3850,7 @@ class iter_impl : public std::iterator<std::bidirectional_iterator_tag, BasicJso @brief post-increment (it++) @pre The iterator is initialized; i.e. `m_object != nullptr`. */ - iter_impl operator++(int) + iter_impl const operator++(int) { auto result = *this; ++(*this); @@ -3898,7 +3893,7 @@ class iter_impl : public std::iterator<std::bidirectional_iterator_tag, BasicJso @brief post-decrement (it--) @pre The iterator is initialized; i.e. `m_object != nullptr`. */ - iter_impl operator--(int) + iter_impl const operator--(int) { auto result = *this; --(*this); @@ -4291,7 +4286,7 @@ class json_reverse_iterator : public std::reverse_iterator<Base> { public: using difference_type = std::ptrdiff_t; - /// shortcut to the reverse iterator adaptor + /// shortcut to the reverse iterator adapter using base_iterator = std::reverse_iterator<Base>; /// the reference type for the pointed-to element using reference = typename Base::reference; @@ -4304,7 +4299,7 @@ class json_reverse_iterator : public std::reverse_iterator<Base> json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {} /// post-increment (it++) - json_reverse_iterator operator++(int) + json_reverse_iterator const operator++(int) { return static_cast<json_reverse_iterator>(base_iterator::operator++(1)); } @@ -4316,7 +4311,7 @@ class json_reverse_iterator : public std::reverse_iterator<Base> } /// post-decrement (it--) - json_reverse_iterator operator--(int) + json_reverse_iterator const operator--(int) { return static_cast<json_reverse_iterator>(base_iterator::operator--(1)); } @@ -4576,12 +4571,12 @@ class binary_reader case 0x07: case 0x08: case 0x09: - case 0x0a: - case 0x0b: - case 0x0c: - case 0x0d: - case 0x0e: - case 0x0f: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: case 0x10: case 0x11: case 0x12: @@ -4598,10 +4593,10 @@ class binary_reader case 0x19: // Unsigned integer (two-byte uint16_t follows) return get_number<uint16_t>(); - case 0x1a: // Unsigned integer (four-byte uint32_t follows) + case 0x1A: // Unsigned integer (four-byte uint32_t follows) return get_number<uint32_t>(); - case 0x1b: // Unsigned integer (eight-byte uint64_t follows) + case 0x1B: // Unsigned integer (eight-byte uint64_t follows) return get_number<uint64_t>(); // Negative integer -1-0x00..-1-0x17 (-1..-24) @@ -4615,12 +4610,12 @@ class binary_reader case 0x27: case 0x28: case 0x29: - case 0x2a: - case 0x2b: - case 0x2c: - case 0x2d: - case 0x2e: - case 0x2f: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: case 0x30: case 0x31: case 0x32: @@ -4642,12 +4637,12 @@ class binary_reader return static_cast<number_integer_t>(-1) - get_number<uint16_t>(); } - case 0x3a: // Negative integer -1-n (four-byte uint32_t follows) + case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) { return static_cast<number_integer_t>(-1) - get_number<uint32_t>(); } - case 0x3b: // Negative integer -1-n (eight-byte uint64_t follows) + case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) { return static_cast<number_integer_t>(-1) - static_cast<number_integer_t>(get_number<uint64_t>()); @@ -4664,12 +4659,12 @@ class binary_reader case 0x67: case 0x68: case 0x69: - case 0x6a: - case 0x6b: - case 0x6c: - case 0x6d: - case 0x6e: - case 0x6f: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: case 0x70: case 0x71: case 0x72: @@ -4680,9 +4675,9 @@ class binary_reader case 0x77: case 0x78: // UTF-8 string (one-byte uint8_t for n follows) case 0x79: // UTF-8 string (two-byte uint16_t for n follow) - case 0x7a: // UTF-8 string (four-byte uint32_t for n follow) - case 0x7b: // UTF-8 string (eight-byte uint64_t for n follow) - case 0x7f: // UTF-8 string (indefinite length) + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + case 0x7F: // UTF-8 string (indefinite length) { return get_cbor_string(); } @@ -4698,12 +4693,12 @@ class binary_reader case 0x87: case 0x88: case 0x89: - case 0x8a: - case 0x8b: - case 0x8c: - case 0x8d: - case 0x8e: - case 0x8f: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: case 0x90: case 0x91: case 0x92: @@ -4713,7 +4708,7 @@ class binary_reader case 0x96: case 0x97: { - return get_cbor_array(current & 0x1f); + return get_cbor_array(current & 0x1F); } case 0x98: // array (one-byte uint8_t for n follows) @@ -4726,20 +4721,20 @@ class binary_reader return get_cbor_array(get_number<uint16_t>()); } - case 0x9a: // array (four-byte uint32_t for n follow) + case 0x9A: // array (four-byte uint32_t for n follow) { return get_cbor_array(get_number<uint32_t>()); } - case 0x9b: // array (eight-byte uint64_t for n follow) + case 0x9B: // array (eight-byte uint64_t for n follow) { return get_cbor_array(get_number<uint64_t>()); } - case 0x9f: // array (indefinite length) + case 0x9F: // array (indefinite length) { BasicJsonType result = value_t::array; - while (get() != 0xff) + while (get() != 0xFF) { result.push_back(parse_cbor_internal(false)); } @@ -4747,58 +4742,58 @@ class binary_reader } // map (0x00..0x17 pairs of data items follow) - case 0xa0: - case 0xa1: - case 0xa2: - case 0xa3: - case 0xa4: - case 0xa5: - case 0xa6: - case 0xa7: - case 0xa8: - case 0xa9: - case 0xaa: - case 0xab: - case 0xac: - case 0xad: - case 0xae: - case 0xaf: - case 0xb0: - case 0xb1: - case 0xb2: - case 0xb3: - case 0xb4: - case 0xb5: - case 0xb6: - case 0xb7: - { - return get_cbor_object(current & 0x1f); - } - - case 0xb8: // map (one-byte uint8_t for n follows) + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + { + return get_cbor_object(current & 0x1F); + } + + case 0xB8: // map (one-byte uint8_t for n follows) { return get_cbor_object(get_number<uint8_t>()); } - case 0xb9: // map (two-byte uint16_t for n follow) + case 0xB9: // map (two-byte uint16_t for n follow) { return get_cbor_object(get_number<uint16_t>()); } - case 0xba: // map (four-byte uint32_t for n follow) + case 0xBA: // map (four-byte uint32_t for n follow) { return get_cbor_object(get_number<uint32_t>()); } - case 0xbb: // map (eight-byte uint64_t for n follow) + case 0xBB: // map (eight-byte uint64_t for n follow) { return get_cbor_object(get_number<uint64_t>()); } - case 0xbf: // map (indefinite length) + case 0xBF: // map (indefinite length) { BasicJsonType result = value_t::object; - while (get() != 0xff) + while (get() != 0xFF) { auto key = get_cbor_string(); result[key] = parse_cbor_internal(); @@ -4806,22 +4801,22 @@ class binary_reader return result; } - case 0xf4: // false + case 0xF4: // false { return false; } - case 0xf5: // true + case 0xF5: // true { return true; } - case 0xf6: // null + case 0xF6: // null { return value_t::null; } - case 0xf9: // Half-Precision Float (two-byte IEEE 754) + case 0xF9: // Half-Precision Float (two-byte IEEE 754) { const int byte1 = get(); check_eof(); @@ -4837,8 +4832,8 @@ class binary_reader // half-precision floating-point numbers in the C language // is shown in Fig. 3. const int half = (byte1 << 8) + byte2; - const int exp = (half >> 10) & 0x1f; - const int mant = half & 0x3ff; + const int exp = (half >> 10) & 0x1F; + const int mant = half & 0x3FF; double val; if (exp == 0) { @@ -4856,12 +4851,12 @@ class binary_reader return (half & 0x8000) != 0 ? -val : val; } - case 0xfa: // Single-Precision Float (four-byte IEEE 754) + case 0xFA: // Single-Precision Float (four-byte IEEE 754) { return get_number<float>(); } - case 0xfb: // Double-Precision Float (eight-byte IEEE 754) + case 0xFB: // Double-Precision Float (eight-byte IEEE 754) { return get_number<double>(); } @@ -4869,7 +4864,7 @@ class binary_reader default: // anything else (0xFF is handled inside the other types) { std::stringstream ss; - ss << std::setw(2) << std::setfill('0') << std::hex << current; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; JSON_THROW(parse_error::create(112, chars_read, "error reading CBOR; last byte: 0x" + ss.str())); } } @@ -4894,12 +4889,12 @@ class binary_reader case 0x07: case 0x08: case 0x09: - case 0x0a: - case 0x0b: - case 0x0c: - case 0x0d: - case 0x0e: - case 0x0f: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: case 0x10: case 0x11: case 0x12: @@ -4910,12 +4905,12 @@ class binary_reader case 0x17: case 0x18: case 0x19: - case 0x1a: - case 0x1b: - case 0x1c: - case 0x1d: - case 0x1e: - case 0x1f: + case 0x1A: + case 0x1B: + case 0x1C: + case 0x1D: + case 0x1E: + case 0x1F: case 0x20: case 0x21: case 0x22: @@ -4926,12 +4921,12 @@ class binary_reader case 0x27: case 0x28: case 0x29: - case 0x2a: - case 0x2b: - case 0x2c: - case 0x2d: - case 0x2e: - case 0x2f: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: case 0x30: case 0x31: case 0x32: @@ -4942,12 +4937,12 @@ class binary_reader case 0x37: case 0x38: case 0x39: - case 0x3a: - case 0x3b: - case 0x3c: - case 0x3d: - case 0x3e: - case 0x3f: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: case 0x40: case 0x41: case 0x42: @@ -4958,12 +4953,12 @@ class binary_reader case 0x47: case 0x48: case 0x49: - case 0x4a: - case 0x4b: - case 0x4c: - case 0x4d: - case 0x4e: - case 0x4f: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: case 0x50: case 0x51: case 0x52: @@ -4974,12 +4969,12 @@ class binary_reader case 0x57: case 0x58: case 0x59: - case 0x5a: - case 0x5b: - case 0x5c: - case 0x5d: - case 0x5e: - case 0x5f: + case 0x5A: + case 0x5B: + case 0x5C: + case 0x5D: + case 0x5E: + case 0x5F: case 0x60: case 0x61: case 0x62: @@ -4990,12 +4985,12 @@ class binary_reader case 0x67: case 0x68: case 0x69: - case 0x6a: - case 0x6b: - case 0x6c: - case 0x6d: - case 0x6e: - case 0x6f: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: case 0x70: case 0x71: case 0x72: @@ -5006,12 +5001,12 @@ class binary_reader case 0x77: case 0x78: case 0x79: - case 0x7a: - case 0x7b: - case 0x7c: - case 0x7d: - case 0x7e: - case 0x7f: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: return static_cast<number_unsigned_t>(current); // fixmap @@ -5025,14 +5020,14 @@ class binary_reader case 0x87: case 0x88: case 0x89: - case 0x8a: - case 0x8b: - case 0x8c: - case 0x8d: - case 0x8e: - case 0x8f: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: { - return get_msgpack_object(current & 0x0f); + return get_msgpack_object(current & 0x0F); } // fixarray @@ -5046,154 +5041,154 @@ class binary_reader case 0x97: case 0x98: case 0x99: - case 0x9a: - case 0x9b: - case 0x9c: - case 0x9d: - case 0x9e: - case 0x9f: + case 0x9A: + case 0x9B: + case 0x9C: + case 0x9D: + case 0x9E: + case 0x9F: { - return get_msgpack_array(current & 0x0f); + return get_msgpack_array(current & 0x0F); } // fixstr - case 0xa0: - case 0xa1: - case 0xa2: - case 0xa3: - case 0xa4: - case 0xa5: - case 0xa6: - case 0xa7: - case 0xa8: - case 0xa9: - case 0xaa: - case 0xab: - case 0xac: - case 0xad: - case 0xae: - case 0xaf: - case 0xb0: - case 0xb1: - case 0xb2: - case 0xb3: - case 0xb4: - case 0xb5: - case 0xb6: - case 0xb7: - case 0xb8: - case 0xb9: - case 0xba: - case 0xbb: - case 0xbc: - case 0xbd: - case 0xbe: - case 0xbf: + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: return get_msgpack_string(); - case 0xc0: // nil + case 0xC0: // nil return value_t::null; - case 0xc2: // false + case 0xC2: // false return false; - case 0xc3: // true + case 0xC3: // true return true; - case 0xca: // float 32 + case 0xCA: // float 32 return get_number<float>(); - case 0xcb: // float 64 + case 0xCB: // float 64 return get_number<double>(); - case 0xcc: // uint 8 + case 0xCC: // uint 8 return get_number<uint8_t>(); - case 0xcd: // uint 16 + case 0xCD: // uint 16 return get_number<uint16_t>(); - case 0xce: // uint 32 + case 0xCE: // uint 32 return get_number<uint32_t>(); - case 0xcf: // uint 64 + case 0xCF: // uint 64 return get_number<uint64_t>(); - case 0xd0: // int 8 + case 0xD0: // int 8 return get_number<int8_t>(); - case 0xd1: // int 16 + case 0xD1: // int 16 return get_number<int16_t>(); - case 0xd2: // int 32 + case 0xD2: // int 32 return get_number<int32_t>(); - case 0xd3: // int 64 + case 0xD3: // int 64 return get_number<int64_t>(); - case 0xd9: // str 8 - case 0xda: // str 16 - case 0xdb: // str 32 + case 0xD9: // str 8 + case 0xDA: // str 16 + case 0xDB: // str 32 return get_msgpack_string(); - case 0xdc: // array 16 + case 0xDC: // array 16 { return get_msgpack_array(get_number<uint16_t>()); } - case 0xdd: // array 32 + case 0xDD: // array 32 { return get_msgpack_array(get_number<uint32_t>()); } - case 0xde: // map 16 + case 0xDE: // map 16 { return get_msgpack_object(get_number<uint16_t>()); } - case 0xdf: // map 32 + case 0xDF: // map 32 { return get_msgpack_object(get_number<uint32_t>()); } // positive fixint - case 0xe0: - case 0xe1: - case 0xe2: - case 0xe3: - case 0xe4: - case 0xe5: - case 0xe6: - case 0xe7: - case 0xe8: - case 0xe9: - case 0xea: - case 0xeb: - case 0xec: - case 0xed: - case 0xee: - case 0xef: - case 0xf0: - case 0xf1: - case 0xf2: - case 0xf3: - case 0xf4: - case 0xf5: - case 0xf6: - case 0xf7: - case 0xf8: - case 0xf9: - case 0xfa: - case 0xfb: - case 0xfc: - case 0xfd: - case 0xfe: - case 0xff: + case 0xE0: + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xED: + case 0xEE: + case 0xEF: + case 0xF0: + case 0xF1: + case 0xF2: + case 0xF3: + case 0xF4: + case 0xF5: + case 0xF6: + case 0xF7: + case 0xF8: + case 0xF9: + case 0xFA: + case 0xFB: + case 0xFC: + case 0xFD: + case 0xFE: + case 0xFF: return static_cast<int8_t>(current); default: // anything else { std::stringstream ss; - ss << std::setw(2) << std::setfill('0') << std::hex << current; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; JSON_THROW(parse_error::create(112, chars_read, "error reading MessagePack; last byte: 0x" + ss.str())); } @@ -5309,12 +5304,12 @@ class binary_reader case 0x67: case 0x68: case 0x69: - case 0x6a: - case 0x6b: - case 0x6c: - case 0x6d: - case 0x6e: - case 0x6f: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: case 0x70: case 0x71: case 0x72: @@ -5324,7 +5319,7 @@ class binary_reader case 0x76: case 0x77: { - return get_string(current & 0x1f); + return get_string(current & 0x1F); } case 0x78: // UTF-8 string (one-byte uint8_t for n follows) @@ -5337,20 +5332,20 @@ class binary_reader return get_string(get_number<uint16_t>()); } - case 0x7a: // UTF-8 string (four-byte uint32_t for n follow) + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) { return get_string(get_number<uint32_t>()); } - case 0x7b: // UTF-8 string (eight-byte uint64_t for n follow) + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) { return get_string(get_number<uint64_t>()); } - case 0x7f: // UTF-8 string (indefinite length) + case 0x7F: // UTF-8 string (indefinite length) { std::string result; - while (get() != 0xff) + while (get() != 0xFF) { check_eof(); result.push_back(static_cast<char>(current)); @@ -5361,7 +5356,7 @@ class binary_reader default: { std::stringstream ss; - ss << std::setw(2) << std::setfill('0') << std::hex << current; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; JSON_THROW(parse_error::create(113, chars_read, "expected a CBOR string; last byte: 0x" + ss.str())); } } @@ -5412,53 +5407,53 @@ class binary_reader switch (current) { // fixstr - case 0xa0: - case 0xa1: - case 0xa2: - case 0xa3: - case 0xa4: - case 0xa5: - case 0xa6: - case 0xa7: - case 0xa8: - case 0xa9: - case 0xaa: - case 0xab: - case 0xac: - case 0xad: - case 0xae: - case 0xaf: - case 0xb0: - case 0xb1: - case 0xb2: - case 0xb3: - case 0xb4: - case 0xb5: - case 0xb6: - case 0xb7: - case 0xb8: - case 0xb9: - case 0xba: - case 0xbb: - case 0xbc: - case 0xbd: - case 0xbe: - case 0xbf: - { - return get_string(current & 0x1f); - } - - case 0xd9: // str 8 + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + { + return get_string(current & 0x1F); + } + + case 0xD9: // str 8 { return get_string(get_number<uint8_t>()); } - case 0xda: // str 16 + case 0xDA: // str 16 { return get_string(get_number<uint16_t>()); } - case 0xdb: // str 32 + case 0xDB: // str 32 { return get_string(get_number<uint32_t>()); } @@ -5466,7 +5461,7 @@ class binary_reader default: { std::stringstream ss; - ss << std::setw(2) << std::setfill('0') << std::hex << current; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; JSON_THROW(parse_error::create(113, chars_read, "expected a MessagePack string; last byte: 0x" + ss.str())); } @@ -5562,15 +5557,15 @@ class binary_writer { case value_t::null: { - oa->write_character(static_cast<CharType>(0xf6)); + oa->write_character(static_cast<CharType>(0xF6)); break; } case value_t::boolean: { oa->write_character(j.m_value.boolean - ? static_cast<CharType>(0xf5) - : static_cast<CharType>(0xf4)); + ? static_cast<CharType>(0xF5) + : static_cast<CharType>(0xF4)); break; } @@ -5597,12 +5592,12 @@ class binary_writer } else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0x1a)); + oa->write_character(static_cast<CharType>(0x1A)); write_number(static_cast<uint32_t>(j.m_value.number_integer)); } else { - oa->write_character(static_cast<CharType>(0x1b)); + oa->write_character(static_cast<CharType>(0x1B)); write_number(static_cast<uint64_t>(j.m_value.number_integer)); } } @@ -5627,12 +5622,12 @@ class binary_writer } else if (positive_number <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0x3a)); + oa->write_character(static_cast<CharType>(0x3A)); write_number(static_cast<uint32_t>(positive_number)); } else { - oa->write_character(static_cast<CharType>(0x3b)); + oa->write_character(static_cast<CharType>(0x3B)); write_number(static_cast<uint64_t>(positive_number)); } } @@ -5657,12 +5652,12 @@ class binary_writer } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0x1a)); + oa->write_character(static_cast<CharType>(0x1A)); write_number(static_cast<uint32_t>(j.m_value.number_unsigned)); } else { - oa->write_character(static_cast<CharType>(0x1b)); + oa->write_character(static_cast<CharType>(0x1B)); write_number(static_cast<uint64_t>(j.m_value.number_unsigned)); } break; @@ -5670,7 +5665,7 @@ class binary_writer case value_t::number_float: // Double-Precision Float { - oa->write_character(static_cast<CharType>(0xfb)); + oa->write_character(static_cast<CharType>(0xFB)); write_number(j.m_value.number_float); break; } @@ -5683,25 +5678,25 @@ class binary_writer { write_number(static_cast<uint8_t>(0x60 + N)); } - else if (N <= 0xff) + else if (N <= 0xFF) { oa->write_character(static_cast<CharType>(0x78)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 0xffff) + else if (N <= 0xFFFF) { oa->write_character(static_cast<CharType>(0x79)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xffffffff) + else if (N <= 0xFFFFFFFF) { - oa->write_character(static_cast<CharType>(0x7a)); + oa->write_character(static_cast<CharType>(0x7A)); write_number(static_cast<uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= 0xffffffffffffffff) + else if (N <= 0xFFFFFFFFFFFFFFFF) { - oa->write_character(static_cast<CharType>(0x7b)); + oa->write_character(static_cast<CharType>(0x7B)); write_number(static_cast<uint64_t>(N)); } // LCOV_EXCL_STOP @@ -5721,25 +5716,25 @@ class binary_writer { write_number(static_cast<uint8_t>(0x80 + N)); } - else if (N <= 0xff) + else if (N <= 0xFF) { oa->write_character(static_cast<CharType>(0x98)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 0xffff) + else if (N <= 0xFFFF) { oa->write_character(static_cast<CharType>(0x99)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xffffffff) + else if (N <= 0xFFFFFFFF) { - oa->write_character(static_cast<CharType>(0x9a)); + oa->write_character(static_cast<CharType>(0x9A)); write_number(static_cast<uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= 0xffffffffffffffff) + else if (N <= 0xFFFFFFFFFFFFFFFF) { - oa->write_character(static_cast<CharType>(0x9b)); + oa->write_character(static_cast<CharType>(0x9B)); write_number(static_cast<uint64_t>(N)); } // LCOV_EXCL_STOP @@ -5758,27 +5753,27 @@ class binary_writer const auto N = j.m_value.object->size(); if (N <= 0x17) { - write_number(static_cast<uint8_t>(0xa0 + N)); + write_number(static_cast<uint8_t>(0xA0 + N)); } - else if (N <= 0xff) + else if (N <= 0xFF) { - oa->write_character(static_cast<CharType>(0xb8)); + oa->write_character(static_cast<CharType>(0xB8)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 0xffff) + else if (N <= 0xFFFF) { - oa->write_character(static_cast<CharType>(0xb9)); + oa->write_character(static_cast<CharType>(0xB9)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xffffffff) + else if (N <= 0xFFFFFFFF) { - oa->write_character(static_cast<CharType>(0xba)); + oa->write_character(static_cast<CharType>(0xBA)); write_number(static_cast<uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= 0xffffffffffffffff) + else if (N <= 0xFFFFFFFFFFFFFFFF) { - oa->write_character(static_cast<CharType>(0xbb)); + oa->write_character(static_cast<CharType>(0xBB)); write_number(static_cast<uint64_t>(N)); } // LCOV_EXCL_STOP @@ -5806,15 +5801,15 @@ class binary_writer { case value_t::null: // nil { - oa->write_character(static_cast<CharType>(0xc0)); + oa->write_character(static_cast<CharType>(0xC0)); break; } case value_t::boolean: // true and false { oa->write_character(j.m_value.boolean - ? static_cast<CharType>(0xc3) - : static_cast<CharType>(0xc2)); + ? static_cast<CharType>(0xC3) + : static_cast<CharType>(0xC2)); break; } @@ -5833,25 +5828,25 @@ class binary_writer else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) { // uint 8 - oa->write_character(static_cast<CharType>(0xcc)); + oa->write_character(static_cast<CharType>(0xCC)); write_number(static_cast<uint8_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)()) { // uint 16 - oa->write_character(static_cast<CharType>(0xcd)); + oa->write_character(static_cast<CharType>(0xCD)); write_number(static_cast<uint16_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) { // uint 32 - oa->write_character(static_cast<CharType>(0xce)); + oa->write_character(static_cast<CharType>(0xCE)); write_number(static_cast<uint32_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)()) { // uint 64 - oa->write_character(static_cast<CharType>(0xcf)); + oa->write_character(static_cast<CharType>(0xCF)); write_number(static_cast<uint64_t>(j.m_value.number_integer)); } } @@ -5866,28 +5861,28 @@ class binary_writer j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)()) { // int 8 - oa->write_character(static_cast<CharType>(0xd0)); + oa->write_character(static_cast<CharType>(0xD0)); write_number(static_cast<int8_t>(j.m_value.number_integer)); } else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)()) { // int 16 - oa->write_character(static_cast<CharType>(0xd1)); + oa->write_character(static_cast<CharType>(0xD1)); write_number(static_cast<int16_t>(j.m_value.number_integer)); } else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)()) { // int 32 - oa->write_character(static_cast<CharType>(0xd2)); + oa->write_character(static_cast<CharType>(0xD2)); write_number(static_cast<int32_t>(j.m_value.number_integer)); } else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)()) { // int 64 - oa->write_character(static_cast<CharType>(0xd3)); + oa->write_character(static_cast<CharType>(0xD3)); write_number(static_cast<int64_t>(j.m_value.number_integer)); } } @@ -5904,25 +5899,25 @@ class binary_writer else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) { // uint 8 - oa->write_character(static_cast<CharType>(0xcc)); + oa->write_character(static_cast<CharType>(0xCC)); write_number(static_cast<uint8_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)()) { // uint 16 - oa->write_character(static_cast<CharType>(0xcd)); + oa->write_character(static_cast<CharType>(0xCD)); write_number(static_cast<uint16_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) { // uint 32 - oa->write_character(static_cast<CharType>(0xce)); + oa->write_character(static_cast<CharType>(0xCE)); write_number(static_cast<uint32_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)()) { // uint 64 - oa->write_character(static_cast<CharType>(0xcf)); + oa->write_character(static_cast<CharType>(0xCF)); write_number(static_cast<uint64_t>(j.m_value.number_integer)); } break; @@ -5930,7 +5925,7 @@ class binary_writer case value_t::number_float: // float 64 { - oa->write_character(static_cast<CharType>(0xcb)); + oa->write_character(static_cast<CharType>(0xCB)); write_number(j.m_value.number_float); break; } @@ -5942,24 +5937,24 @@ class binary_writer if (N <= 31) { // fixstr - write_number(static_cast<uint8_t>(0xa0 | N)); + write_number(static_cast<uint8_t>(0xA0 | N)); } else if (N <= 255) { // str 8 - oa->write_character(static_cast<CharType>(0xd9)); + oa->write_character(static_cast<CharType>(0xD9)); write_number(static_cast<uint8_t>(N)); } else if (N <= 65535) { // str 16 - oa->write_character(static_cast<CharType>(0xda)); + oa->write_character(static_cast<CharType>(0xDA)); write_number(static_cast<uint16_t>(N)); } else if (N <= 4294967295) { // str 32 - oa->write_character(static_cast<CharType>(0xdb)); + oa->write_character(static_cast<CharType>(0xDB)); write_number(static_cast<uint32_t>(N)); } @@ -5979,16 +5974,16 @@ class binary_writer // fixarray write_number(static_cast<uint8_t>(0x90 | N)); } - else if (N <= 0xffff) + else if (N <= 0xFFFF) { // array 16 - oa->write_character(static_cast<CharType>(0xdc)); + oa->write_character(static_cast<CharType>(0xDC)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xffffffff) + else if (N <= 0xFFFFFFFF) { // array 32 - oa->write_character(static_cast<CharType>(0xdd)); + oa->write_character(static_cast<CharType>(0xDD)); write_number(static_cast<uint32_t>(N)); } @@ -6007,18 +6002,18 @@ class binary_writer if (N <= 15) { // fixmap - write_number(static_cast<uint8_t>(0x80 | (N & 0xf))); + write_number(static_cast<uint8_t>(0x80 | (N & 0xF))); } else if (N <= 65535) { // map 16 - oa->write_character(static_cast<CharType>(0xde)); + oa->write_character(static_cast<CharType>(0xDE)); write_number(static_cast<uint16_t>(N)); } else if (N <= 4294967295) { // map 32 - oa->write_character(static_cast<CharType>(0xdf)); + oa->write_character(static_cast<CharType>(0xDF)); write_number(static_cast<uint32_t>(N)); } @@ -6363,9 +6358,9 @@ class serializer case 0x05: case 0x06: case 0x07: - case 0x0b: - case 0x0e: - case 0x0f: + case 0x0B: + case 0x0E: + case 0x0F: case 0x10: case 0x11: case 0x12: @@ -6376,12 +6371,12 @@ class serializer case 0x17: case 0x18: case 0x19: - case 0x1a: - case 0x1b: - case 0x1c: - case 0x1d: - case 0x1e: - case 0x1f: + case 0x1A: + case 0x1B: + case 0x1C: + case 0x1D: + case 0x1E: + case 0x1F: { // from c (1 byte) to \uxxxx (6 bytes) res += 5; @@ -6393,12 +6388,8 @@ class serializer if (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F)) { const auto bytes = bytes_following(static_cast<uint8_t>(s[i])); - if (bytes == std::string::npos) - { - // invalid characters are treated as is, so no - // additional space will be used - break; - } + // invalid characters will be detected by throw_if_invalid_utf8 + assert (bytes != std::string::npos); if (bytes == 3) { @@ -6492,6 +6483,8 @@ class serializer */ void dump_escaped(const string_t& s, const bool ensure_ascii) const { + throw_if_invalid_utf8(s); + const auto space = extra_space(s, ensure_ascii); if (space == 0) { @@ -6514,7 +6507,7 @@ class serializer break; } - case '\\': // reverse solidus (0x5c) + case '\\': // reverse solidus (0x5C) { // nothing to change pos += 2; @@ -6528,21 +6521,21 @@ class serializer break; } - case '\f': // formfeed (0x0c) + case '\f': // formfeed (0x0C) { result[pos + 1] = 'f'; pos += 2; break; } - case '\n': // newline (0x0a) + case '\n': // newline (0x0A) { result[pos + 1] = 'n'; pos += 2; break; } - case '\r': // carriage return (0x0d) + case '\r': // carriage return (0x0D) { result[pos + 1] = 'r'; pos += 2; @@ -6564,21 +6557,18 @@ class serializer (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F))) { const auto bytes = bytes_following(static_cast<uint8_t>(s[i])); - if (bytes == std::string::npos) - { - // copy invalid character as is - result[pos++] = s[i]; - break; - } + // invalid characters will be detected by throw_if_invalid_utf8 + assert (bytes != std::string::npos); // check that the additional bytes are present assert(i + bytes < s.size()); - // to useย \uxxxx escaping, we first need to caluclate + // to use \uxxxx escaping, we first need to calculate // the codepoint from the UTF-8 bytes int codepoint = 0; - assert(0 <= bytes and bytes <= 3); + // bytes is unsigned type: + assert(bytes <= 3); switch (bytes) { case 0: @@ -6641,11 +6631,10 @@ class serializer @param[in] x integer number (signed or unsigned) to dump @tparam NumberType either @a number_integer_t or @a number_unsigned_t */ - template < - typename NumberType, - detail::enable_if_t<std::is_same<NumberType, number_unsigned_t>::value or - std::is_same<NumberType, number_integer_t>::value, - int> = 0 > + template<typename NumberType, detail::enable_if_t< + std::is_same<NumberType, number_unsigned_t>::value or + std::is_same<NumberType, number_integer_t>::value, + int> = 0> void dump_integer(NumberType x) { // special case for "0" @@ -6743,6 +6732,87 @@ class serializer } } + /*! + @brief check whether a string is UTF-8 encoded + + The function checks each byte of a string whether it is UTF-8 encoded. The + result of the check is stored in the @a state parameter. The function must + be called initially with state 0 (accept). State 1 means the string must + be rejected, because the current byte is not allowed. If the string is + completely processed, but the state is non-zero, the string ended + prematurely; that is, the last byte indicated more bytes should have + followed. + + @param[in,out] state the state of the decoding + @param[in] byte next byte to decode + + @note The function has been edited: a std::array is used and the code + point is not calculated. + + @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de> + @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + */ + static void decode(uint8_t& state, const uint8_t byte) + { + static const std::array<uint8_t, 400> utf8d = + { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF + 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF + 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF + 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF + 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 + 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 + 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 + 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 + } + }; + + const uint8_t type = utf8d[byte]; + state = utf8d[256u + state * 16u + type]; + } + + /*! + @brief throw an exception if a string is not UTF-8 encoded + + @param[in] str UTF-8 string to check + @throw type_error.316 if passed string is not UTF-8 encoded + + @since version 3.0.0 + */ + static void throw_if_invalid_utf8(const std::string& str) + { + // start with state 0 (= accept) + uint8_t state = 0; + + for (size_t i = 0; i < str.size(); ++i) + { + const auto byte = static_cast<uint8_t>(str[i]); + decode(state, byte); + if (state == 1) + { + // state 1 means reject + std::stringstream ss; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(byte); + JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + ss.str())); + } + } + + if (state != 0) + { + // we finish reading, but do not accept: string was incomplete + std::stringstream ss; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(static_cast<uint8_t>(str.back())); + JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + ss.str())); + } + } + private: /// the output of the serializer output_adapter_t<char> o = nullptr; @@ -6771,27 +6841,20 @@ class json_ref using value_type = BasicJsonType; json_ref(value_type&& value) - : owned_value(std::move(value)), - value_ref(&owned_value), - is_rvalue(true) + : owned_value(std::move(value)), value_ref(&owned_value), is_rvalue(true) {} json_ref(const value_type& value) - : value_ref(const_cast<value_type*>(&value)), - is_rvalue(false) + : value_ref(const_cast<value_type*>(&value)), is_rvalue(false) {} json_ref(std::initializer_list<json_ref> init) - : owned_value(init), - value_ref(&owned_value), - is_rvalue(true) + : owned_value(init), value_ref(&owned_value), is_rvalue(true) {} - template <class... Args> + template<class... Args> json_ref(Args&& ... args) - : owned_value(std::forward<Args>(args)...), - value_ref(&owned_value), - is_rvalue(true) + : owned_value(std::forward<Args>(args)...), value_ref(&owned_value), is_rvalue(true) {} // class should be movable only @@ -6949,6 +7012,27 @@ class json_pointer return to_string(); } + /*! + @param[in] s reference token to be converted into an array index + + @return integer representation of @a s + + @throw out_of_range.404 if string @a s could not be converted to an integer + */ + static int array_index(const std::string& s) + { + size_t processed_chars = 0; + const int res = std::stoi(s, &processed_chars); + + // check if the string was completely read + if (JSON_UNLIKELY(processed_chars != s.size())) + { + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'")); + } + + return res; + } + private: /*! @brief remove and return last reference pointer @@ -6984,7 +7068,6 @@ class json_pointer return result; } - /*! @brief create and return a reference to the pointed to value @@ -7320,11 +7403,11 @@ class basic_json public: using value_t = detail::value_t; - // forward declarations + /// @copydoc nlohmann::json_pointer using json_pointer = ::nlohmann::json_pointer; template<typename T, typename SFINAE> using json_serializer = JSONSerializer<T, SFINAE>; - + /// helper type for initializer lists of basic_json values using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>; //////////////// @@ -7436,7 +7519,7 @@ class basic_json result["url"] = "https://github.com/nlohmann/json"; result["version"] = { - {"string", "2.1.1"}, {"major", 2}, {"minor", 1}, {"patch", 1} + {"string", "3.0.1"}, {"major", 3}, {"minor", 0}, {"patch", 1} }; #ifdef _WIN32 @@ -7489,6 +7572,14 @@ class basic_json /// the template arguments passed to class @ref basic_json. /// @{ +#if defined(JSON_HAS_CPP_14) + // Use transparent comparator if possible, combined with perfect forwarding + // on find() and count() calls prevents unnecessary string construction. + using object_comparator_t = std::less<>; +#else + using object_comparator_t = std::less<StringType>; +#endif + /*! @brief a type for an object @@ -7572,14 +7663,6 @@ class basic_json 7159](http://rfc7159.net/rfc7159), because any order implements the specified "unordered" nature of JSON objects. */ - -#if defined(JSON_HAS_CPP_14) - // Use transparent comparator if possible, combined with perfect forwarding - // on find() and count() calls prevents unnecessary string construction. - using object_comparator_t = std::less<>; -#else - using object_comparator_t = std::less<StringType>; -#endif using object_t = ObjectType<StringType, basic_json, object_comparator_t, @@ -7931,12 +8014,14 @@ class basic_json static T* create(Args&& ... args) { AllocatorType<T> alloc; + using AllocatorTraits = std::allocator_traits<AllocatorType<T>>; + auto deleter = [&](T * object) { - alloc.deallocate(object, 1); + AllocatorTraits::deallocate(alloc, object, 1); }; - std::unique_ptr<T, decltype(deleter)> object(alloc.allocate(1), deleter); - alloc.construct(object.get(), std::forward<Args>(args)...); + std::unique_ptr<T, decltype(deleter)> object(AllocatorTraits::allocate(alloc, 1), deleter); + AllocatorTraits::construct(alloc, object.get(), std::forward<Args>(args)...); assert(object != nullptr); return object.release(); } @@ -8054,7 +8139,7 @@ class basic_json object = nullptr; // silence warning, see #821 if (JSON_UNLIKELY(t == value_t::null)) { - JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 2.1.1")); // LCOV_EXCL_LINE + JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.0.1")); // LCOV_EXCL_LINE } break; } @@ -8104,24 +8189,24 @@ class basic_json case value_t::object: { AllocatorType<object_t> alloc; - alloc.destroy(object); - alloc.deallocate(object, 1); + std::allocator_traits<decltype(alloc)>::destroy(alloc, object); + std::allocator_traits<decltype(alloc)>::deallocate(alloc, object, 1); break; } case value_t::array: { AllocatorType<array_t> alloc; - alloc.destroy(array); - alloc.deallocate(array, 1); + std::allocator_traits<decltype(alloc)>::destroy(alloc, array); + std::allocator_traits<decltype(alloc)>::deallocate(alloc, array, 1); break; } case value_t::string: { AllocatorType<string_t> alloc; - alloc.destroy(string); - alloc.deallocate(string, 1); + std::allocator_traits<decltype(alloc)>::destroy(alloc, string); + std::allocator_traits<decltype(alloc)>::deallocate(alloc, string, 1); break; } @@ -8154,6 +8239,21 @@ class basic_json // JSON parser callback // ////////////////////////// + /*! + @brief parser event types + + The parser callback distinguishes the following events: + - `object_start`: the parser read `{` and started to process a JSON object + - `key`: the parser read a key of a value in an object + - `object_end`: the parser read `}` and finished processing a JSON object + - `array_start`: the parser read `[` and started to process a JSON array + - `array_end`: the parser read `]` and finished processing a JSON array + - `value`: the parser finished reading a JSON value + + @image html callback_events.png "Example when certain parse events are triggered" + + @sa @ref parser_callback_t for more information and examples + */ using parse_event_t = typename parser::parse_event_t; /*! @@ -8280,7 +8380,7 @@ class basic_json @brief create a JSON value This is a "catch all" constructor for all compatible JSON types; that is, - types for which a `to_json()` method exsits. The constructor forwards the + types for which a `to_json()` method exists. The constructor forwards the parameter @a val to that method (to `json_serializer<U>::to_json` method with `U = uncvref_t<CompatibleType>`, to be exact). @@ -8952,11 +9052,14 @@ class basic_json @param[in] indent_char The character to use for indentation if @a indent is greater than `0`. The default is ` ` (space). @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters - in the output are escaped with \uXXXX sequences, and the result consists + in the output are escaped with `\uXXXX` sequences, and the result consists of ASCII characters only. @return string containing the serialization of the JSON value + @throw type_error.316 if a string stored inside the JSON value is not + UTF-8 encoded + @complexity Linear. @exceptionsafety Strong guarantee: if an exception is thrown, there are no @@ -8968,8 +9071,8 @@ class basic_json @see https://docs.python.org/2/library/json.html#json.dump - @since version 1.0.0; indentation character @a indent_char and option - @a ensure_ascii added in version 3.0.0 + @since version 1.0.0; indentation character @a indent_char, option + @a ensure_ascii and exceptions added in version 3.0.0 */ string_t dump(const int indent = -1, const char indent_char = ' ', const bool ensure_ascii = false) const @@ -9003,7 +9106,7 @@ class basic_json string | value_t::string number (integer) | value_t::number_integer number (unsigned integer) | value_t::number_unsigned - number (foating-point) | value_t::number_float + number (floating-point) | value_t::number_float object | value_t::object array | value_t::array discarded | value_t::discarded @@ -9507,11 +9610,9 @@ class basic_json @since version 2.1.0 */ - template < - typename BasicJsonType, - detail::enable_if_t<std::is_same<typename std::remove_const<BasicJsonType>::type, - basic_json_t>::value, - int> = 0 > + template<typename BasicJsonType, detail::enable_if_t< + std::is_same<typename std::remove_const<BasicJsonType>::type, basic_json_t>::value, + int> = 0> basic_json get() const { return *this; @@ -9556,14 +9657,12 @@ class basic_json @since version 2.1.0 */ - template < - typename ValueTypeCV, - typename ValueType = detail::uncvref_t<ValueTypeCV>, - detail::enable_if_t < - not std::is_same<basic_json_t, ValueType>::value and - detail::has_from_json<basic_json_t, ValueType>::value and - not detail::has_non_default_from_json<basic_json_t, ValueType>::value, - int > = 0 > + template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>, + detail::enable_if_t < + not std::is_same<basic_json_t, ValueType>::value and + detail::has_from_json<basic_json_t, ValueType>::value and + not detail::has_non_default_from_json<basic_json_t, ValueType>::value, + int> = 0> ValueType get() const noexcept(noexcept( JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), std::declval<ValueType&>()))) { @@ -9611,12 +9710,10 @@ class basic_json @since version 2.1.0 */ - template < - typename ValueTypeCV, - typename ValueType = detail::uncvref_t<ValueTypeCV>, - detail::enable_if_t<not std::is_same<basic_json_t, ValueType>::value and - detail::has_non_default_from_json<basic_json_t, - ValueType>::value, int> = 0 > + template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>, + detail::enable_if_t<not std::is_same<basic_json_t, ValueType>::value and + detail::has_non_default_from_json<basic_json_t, ValueType>::value, + int> = 0> ValueType get() const noexcept(noexcept( JSONSerializer<ValueTypeCV>::from_json(std::declval<const basic_json_t&>()))) { @@ -10110,7 +10207,7 @@ class basic_json @return const reference to the element at index @a idx - @throw type_error.305 if the JSON value is not an array; in that cases, + @throw type_error.305 if the JSON value is not an array; in that case, using the [] operator with an index makes no sense. @complexity Constant. @@ -10193,7 +10290,7 @@ class basic_json @pre The element with key @a key must exist. **This precondition is enforced with an assertion.** - @throw type_error.305 if the JSON value is not an object; in that cases, + @throw type_error.305 if the JSON value is not an object; in that case, using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -10282,7 +10379,7 @@ class basic_json @pre The element with key @a key must exist. **This precondition is enforced with an assertion.** - @throw type_error.305 if the JSON value is not an object; in that cases, + @throw type_error.305 if the JSON value is not an object; in that case, using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -10342,7 +10439,7 @@ class basic_json @return copy of the element at key @a key or @a default_value if @a key is not found - @throw type_error.306 if the JSON value is not an objec; in that cases, + @throw type_error.306 if the JSON value is not an object; in that case, using `value()` with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -10415,7 +10512,7 @@ class basic_json @return copy of the element at key @a key or @a default_value if @a key is not found - @throw type_error.306 if the JSON value is not an objec; in that cases, + @throw type_error.306 if the JSON value is not an objec; in that case, using `value()` with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -10619,8 +10716,8 @@ class basic_json if (is_string()) { AllocatorType<string_t> alloc; - alloc.destroy(m_value.string); - alloc.deallocate(m_value.string, 1); + std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.string); + std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.string, 1); m_value.string = nullptr; } @@ -10725,8 +10822,8 @@ class basic_json if (is_string()) { AllocatorType<string_t> alloc; - alloc.destroy(m_value.string); - alloc.deallocate(m_value.string, 1); + std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.string); + std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.string, 1); m_value.string = nullptr; } @@ -11220,22 +11317,62 @@ class basic_json reference to the JSON values is returned, so there is no access to the underlying iterator. + For loop without iterator_wrapper: + + @code{cpp} + for (auto it = j_object.begin(); it != j_object.end(); ++it) + { + std::cout << "key: " << it.key() << ", value:" << it.value() << '\n'; + } + @endcode + + Range-based for loop without iterator proxy: + + @code{cpp} + for (auto it : j_object) + { + // "it" is of type json::reference and has no key() member + std::cout << "value: " << it << '\n'; + } + @endcode + + Range-based for loop with iterator proxy: + + @code{cpp} + for (auto it : json::iterator_wrapper(j_object)) + { + std::cout << "key: " << it.key() << ", value:" << it.value() << '\n'; + } + @endcode + + @note When iterating over an array, `key()` will return the index of the + element as string (see example). + + @param[in] ref reference to a JSON value + @return iteration proxy object wrapping @a ref with an interface to use in + range-based for loops + @liveexample{The following code shows how the wrapper is used,iterator_wrapper} + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Constant. + @note The name of this function is not yet final and may change in the future. */ - static iteration_proxy<iterator> iterator_wrapper(reference cont) + static iteration_proxy<iterator> iterator_wrapper(reference ref) { - return iteration_proxy<iterator>(cont); + return iteration_proxy<iterator>(ref); } /*! @copydoc iterator_wrapper(reference) */ - static iteration_proxy<const_iterator> iterator_wrapper(const_reference cont) + static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref) { - return iteration_proxy<const_iterator>(cont); + return iteration_proxy<const_iterator>(ref); } /// @} @@ -12120,7 +12257,7 @@ class basic_json JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(j.type_name()))); } - for (auto it = j.begin(); it != j.end(); ++it) + for (auto it = j.cbegin(); it != j.cend(); ++it) { m_value.object->operator[](it.key()) = it.value(); } @@ -12341,7 +12478,7 @@ class basic_json [comparison function](https://github.com/mariokonrad/marnav/blob/master/src/marnav/math/floatingpoint.hpp#L34-#L39) could be used, for instance @code {.cpp} - template <typename T, typename = typename std::enable_if<std::is_floating_point<T>::value, T>::type> + template<typename T, typename = typename std::enable_if<std::is_floating_point<T>::value, T>::type> inline bool is_same(T a, T b, T epsilon = std::numeric_limits<T>::epsilon()) noexcept { return std::abs(a - b) <= epsilon; @@ -12769,7 +12906,7 @@ class basic_json `std::setw(4)` on @a o sets the indentation level to `4` and the serialization result is the same as calling `dump(4)`. - - The indentation characrer can be controlled with the member variable + - The indentation character can be controlled with the member variable `fill` of the output stream @a o. For instance, the manipulator `std::setfill('\\t')` sets indentation to use a tab character rather than the default space character. @@ -12779,12 +12916,15 @@ class basic_json @return the stream @a o + @throw type_error.316 if a string stored inside the JSON value is not + UTF-8 encoded + @complexity Linear. @liveexample{The example below shows the serialization with different parameters to `width` to adjust the indentation level.,operator_serialize} - @since version 1.0.0; indentaction character added in version 3.0.0 + @since version 1.0.0; indentation character added in version 3.0.0 */ friend std::ostream& operator<<(std::ostream& o, const basic_json& j) { @@ -13124,40 +13264,40 @@ class basic_json JSON value type | value/range | CBOR type | first byte --------------- | ------------------------------------------ | ---------------------------------- | --------------- - null | `null` | Null | 0xf6 - boolean | `true` | True | 0xf5 - boolean | `false` | False | 0xf4 - number_integer | -9223372036854775808..-2147483649 | Negative integer (8 bytes follow) | 0x3b - number_integer | -2147483648..-32769 | Negative integer (4 bytes follow) | 0x3a + null | `null` | Null | 0xF6 + boolean | `true` | True | 0xF5 + boolean | `false` | False | 0xF4 + number_integer | -9223372036854775808..-2147483649 | Negative integer (8 bytes follow) | 0x3B + number_integer | -2147483648..-32769 | Negative integer (4 bytes follow) | 0x3A number_integer | -32768..-129 | Negative integer (2 bytes follow) | 0x39 number_integer | -128..-25 | Negative integer (1 byte follow) | 0x38 number_integer | -24..-1 | Negative integer | 0x20..0x37 number_integer | 0..23 | Integer | 0x00..0x17 number_integer | 24..255 | Unsigned integer (1 byte follow) | 0x18 number_integer | 256..65535 | Unsigned integer (2 bytes follow) | 0x19 - number_integer | 65536..4294967295 | Unsigned integer (4 bytes follow) | 0x1a - number_integer | 4294967296..18446744073709551615 | Unsigned integer (8 bytes follow) | 0x1b + number_integer | 65536..4294967295 | Unsigned integer (4 bytes follow) | 0x1A + number_integer | 4294967296..18446744073709551615 | Unsigned integer (8 bytes follow) | 0x1B number_unsigned | 0..23 | Integer | 0x00..0x17 number_unsigned | 24..255 | Unsigned integer (1 byte follow) | 0x18 number_unsigned | 256..65535 | Unsigned integer (2 bytes follow) | 0x19 - number_unsigned | 65536..4294967295 | Unsigned integer (4 bytes follow) | 0x1a - number_unsigned | 4294967296..18446744073709551615 | Unsigned integer (8 bytes follow) | 0x1b - number_float | *any value* | Double-Precision Float | 0xfb + number_unsigned | 65536..4294967295 | Unsigned integer (4 bytes follow) | 0x1A + number_unsigned | 4294967296..18446744073709551615 | Unsigned integer (8 bytes follow) | 0x1B + number_float | *any value* | Double-Precision Float | 0xFB string | *length*: 0..23 | UTF-8 string | 0x60..0x77 string | *length*: 23..255 | UTF-8 string (1 byte follow) | 0x78 string | *length*: 256..65535 | UTF-8 string (2 bytes follow) | 0x79 - string | *length*: 65536..4294967295 | UTF-8 string (4 bytes follow) | 0x7a - string | *length*: 4294967296..18446744073709551615 | UTF-8 string (8 bytes follow) | 0x7b + string | *length*: 65536..4294967295 | UTF-8 string (4 bytes follow) | 0x7A + string | *length*: 4294967296..18446744073709551615 | UTF-8 string (8 bytes follow) | 0x7B array | *size*: 0..23 | array | 0x80..0x97 array | *size*: 23..255 | array (1 byte follow) | 0x98 array | *size*: 256..65535 | array (2 bytes follow) | 0x99 - array | *size*: 65536..4294967295 | array (4 bytes follow) | 0x9a - array | *size*: 4294967296..18446744073709551615 | array (8 bytes follow) | 0x9b - object | *size*: 0..23 | map | 0xa0..0xb7 - object | *size*: 23..255 | map (1 byte follow) | 0xb8 - object | *size*: 256..65535 | map (2 bytes follow) | 0xb9 - object | *size*: 65536..4294967295 | map (4 bytes follow) | 0xba - object | *size*: 4294967296..18446744073709551615 | map (8 bytes follow) | 0xbb + array | *size*: 65536..4294967295 | array (4 bytes follow) | 0x9A + array | *size*: 4294967296..18446744073709551615 | array (8 bytes follow) | 0x9B + object | *size*: 0..23 | map | 0xA0..0xB7 + object | *size*: 23..255 | map (1 byte follow) | 0xB8 + object | *size*: 256..65535 | map (2 bytes follow) | 0xB9 + object | *size*: 65536..4294967295 | map (4 bytes follow) | 0xBA + object | *size*: 4294967296..18446744073709551615 | map (8 bytes follow) | 0xBB @note The mapping is **complete** in the sense that any JSON value type can be converted to a CBOR value. @@ -13167,20 +13307,20 @@ class basic_json function which serializes NaN or Infinity to `null`. @note The following CBOR types are not used in the conversion: - - byte strings (0x40..0x5f) - - UTF-8 strings terminated by "break" (0x7f) - - arrays terminated by "break" (0x9f) - - maps terminated by "break" (0xbf) - - date/time (0xc0..0xc1) - - bignum (0xc2..0xc3) - - decimal fraction (0xc4) - - bigfloat (0xc5) - - tagged items (0xc6..0xd4, 0xd8..0xdb) - - expected conversions (0xd5..0xd7) - - simple values (0xe0..0xf3, 0xf8) - - undefined (0xf7) - - half and single-precision floats (0xf9-0xfa) - - break (0xff) + - byte strings (0x40..0x5F) + - UTF-8 strings terminated by "break" (0x7F) + - arrays terminated by "break" (0x9F) + - maps terminated by "break" (0xBF) + - date/time (0xC0..0xC1) + - bignum (0xC2..0xC3) + - decimal fraction (0xC4) + - bigfloat (0xC5) + - tagged items (0xC6..0xD4, 0xD8..0xDB) + - expected conversions (0xD5..0xD7) + - simple values (0xE0..0xF3, 0xF8) + - undefined (0xF7) + - half and single-precision floats (0xF9-0xFA) + - break (0xFF) @param[in] j JSON value to serialize @return MessagePack serialization as byte vector @@ -13226,35 +13366,35 @@ class basic_json JSON value type | value/range | MessagePack type | first byte --------------- | --------------------------------- | ---------------- | ---------- - null | `null` | nil | 0xc0 - boolean | `true` | true | 0xc3 - boolean | `false` | false | 0xc2 - number_integer | -9223372036854775808..-2147483649 | int64 | 0xd3 - number_integer | -2147483648..-32769 | int32 | 0xd2 - number_integer | -32768..-129 | int16 | 0xd1 - number_integer | -128..-33 | int8 | 0xd0 - number_integer | -32..-1 | negative fixint | 0xe0..0xff - number_integer | 0..127 | positive fixint | 0x00..0x7f - number_integer | 128..255 | uint 8 | 0xcc - number_integer | 256..65535 | uint 16 | 0xcd - number_integer | 65536..4294967295 | uint 32 | 0xce - number_integer | 4294967296..18446744073709551615 | uint 64 | 0xcf - number_unsigned | 0..127 | positive fixint | 0x00..0x7f - number_unsigned | 128..255 | uint 8 | 0xcc - number_unsigned | 256..65535 | uint 16 | 0xcd - number_unsigned | 65536..4294967295 | uint 32 | 0xce - number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xcf - number_float | *any value* | float 64 | 0xcb - string | *length*: 0..31 | fixstr | 0xa0..0xbf - string | *length*: 32..255 | str 8 | 0xd9 - string | *length*: 256..65535 | str 16 | 0xda - string | *length*: 65536..4294967295 | str 32 | 0xdb - array | *size*: 0..15 | fixarray | 0x90..0x9f - array | *size*: 16..65535 | array 16 | 0xdc - array | *size*: 65536..4294967295 | array 32 | 0xdd - object | *size*: 0..15 | fix map | 0x80..0x8f - object | *size*: 16..65535 | map 16 | 0xde - object | *size*: 65536..4294967295 | map 32 | 0xdf + null | `null` | nil | 0xC0 + boolean | `true` | true | 0xC3 + boolean | `false` | false | 0xC2 + number_integer | -9223372036854775808..-2147483649 | int64 | 0xD3 + number_integer | -2147483648..-32769 | int32 | 0xD2 + number_integer | -32768..-129 | int16 | 0xD1 + number_integer | -128..-33 | int8 | 0xD0 + number_integer | -32..-1 | negative fixint | 0xE0..0xFF + number_integer | 0..127 | positive fixint | 0x00..0x7F + number_integer | 128..255 | uint 8 | 0xCC + number_integer | 256..65535 | uint 16 | 0xCD + number_integer | 65536..4294967295 | uint 32 | 0xCE + number_integer | 4294967296..18446744073709551615 | uint 64 | 0xCF + number_unsigned | 0..127 | positive fixint | 0x00..0x7F + number_unsigned | 128..255 | uint 8 | 0xCC + number_unsigned | 256..65535 | uint 16 | 0xCD + number_unsigned | 65536..4294967295 | uint 32 | 0xCE + number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xCF + number_float | *any value* | float 64 | 0xCB + string | *length*: 0..31 | fixstr | 0xA0..0xBF + string | *length*: 32..255 | str 8 | 0xD9 + string | *length*: 256..65535 | str 16 | 0xDA + string | *length*: 65536..4294967295 | str 32 | 0xDB + array | *size*: 0..15 | fixarray | 0x90..0x9F + array | *size*: 16..65535 | array 16 | 0xDC + array | *size*: 65536..4294967295 | array 32 | 0xDD + object | *size*: 0..15 | fix map | 0x80..0x8F + object | *size*: 16..65535 | map 16 | 0xDE + object | *size*: 65536..4294967295 | map 32 | 0xDF @note The mapping is **complete** in the sense that any JSON value type can be converted to a MessagePack value. @@ -13265,10 +13405,10 @@ class basic_json - objects with more than 4294967295 elements @note The following MessagePack types are not used in the conversion: - - bin 8 - bin 32 (0xc4..0xc6) - - ext 8 - ext 32 (0xc7..0xc9) - - float 32 (0xca) - - fixext 1 - fixext 16 (0xd4..0xd8) + - bin 8 - bin 32 (0xC4..0xC6) + - ext 8 - ext 32 (0xC7..0xC9) + - float 32 (0xCA) + - fixext 1 - fixext 16 (0xD4..0xD8) @note Any MessagePack output created @ref to_msgpack can be successfully parsed by @ref from_msgpack. @@ -13322,51 +13462,51 @@ class basic_json Integer | number_unsigned | 0x00..0x17 Unsigned integer | number_unsigned | 0x18 Unsigned integer | number_unsigned | 0x19 - Unsigned integer | number_unsigned | 0x1a - Unsigned integer | number_unsigned | 0x1b + Unsigned integer | number_unsigned | 0x1A + Unsigned integer | number_unsigned | 0x1B Negative integer | number_integer | 0x20..0x37 Negative integer | number_integer | 0x38 Negative integer | number_integer | 0x39 - Negative integer | number_integer | 0x3a - Negative integer | number_integer | 0x3b + Negative integer | number_integer | 0x3A + Negative integer | number_integer | 0x3B Negative integer | number_integer | 0x40..0x57 UTF-8 string | string | 0x60..0x77 UTF-8 string | string | 0x78 UTF-8 string | string | 0x79 - UTF-8 string | string | 0x7a - UTF-8 string | string | 0x7b - UTF-8 string | string | 0x7f + UTF-8 string | string | 0x7A + UTF-8 string | string | 0x7B + UTF-8 string | string | 0x7F array | array | 0x80..0x97 array | array | 0x98 array | array | 0x99 - array | array | 0x9a - array | array | 0x9b - array | array | 0x9f - map | object | 0xa0..0xb7 - map | object | 0xb8 - map | object | 0xb9 - map | object | 0xba - map | object | 0xbb - map | object | 0xbf - False | `false` | 0xf4 - True | `true` | 0xf5 - Nill | `null` | 0xf6 - Half-Precision Float | number_float | 0xf9 - Single-Precision Float | number_float | 0xfa - Double-Precision Float | number_float | 0xfb + array | array | 0x9A + array | array | 0x9B + array | array | 0x9F + map | object | 0xA0..0xB7 + map | object | 0xB8 + map | object | 0xB9 + map | object | 0xBA + map | object | 0xBB + map | object | 0xBF + False | `false` | 0xF4 + True | `true` | 0xF5 + Nill | `null` | 0xF6 + Half-Precision Float | number_float | 0xF9 + Single-Precision Float | number_float | 0xFA + Double-Precision Float | number_float | 0xFB @warning The mapping is **incomplete** in the sense that not all CBOR types can be converted to a JSON value. The following CBOR types are not supported and will yield parse errors (parse_error.112): - - byte strings (0x40..0x5f) - - date/time (0xc0..0xc1) - - bignum (0xc2..0xc3) - - decimal fraction (0xc4) - - bigfloat (0xc5) - - tagged items (0xc6..0xd4, 0xd8..0xdb) - - expected conversions (0xd5..0xd7) - - simple values (0xe0..0xf3, 0xf8) - - undefined (0xf7) + - byte strings (0x40..0x5F) + - date/time (0xC0..0xC1) + - bignum (0xC2..0xC3) + - decimal fraction (0xC4) + - bigfloat (0xC5) + - tagged items (0xC6..0xD4, 0xD8..0xDB) + - expected conversions (0xD5..0xD7) + - simple values (0xE0..0xF3, 0xF8) + - undefined (0xF7) @warning CBOR allows map keys of any type, whereas JSON only allows strings as keys in object values. Therefore, CBOR maps with keys @@ -13426,38 +13566,38 @@ class basic_json MessagePack type | JSON value type | first byte ---------------- | --------------- | ---------- - positive fixint | number_unsigned | 0x00..0x7f - fixmap | object | 0x80..0x8f - fixarray | array | 0x90..0x9f - fixstr | string | 0xa0..0xbf - nil | `null` | 0xc0 - false | `false` | 0xc2 - true | `true` | 0xc3 - float 32 | number_float | 0xca - float 64 | number_float | 0xcb - uint 8 | number_unsigned | 0xcc - uint 16 | number_unsigned | 0xcd - uint 32 | number_unsigned | 0xce - uint 64 | number_unsigned | 0xcf - int 8 | number_integer | 0xd0 - int 16 | number_integer | 0xd1 - int 32 | number_integer | 0xd2 - int 64 | number_integer | 0xd3 - str 8 | string | 0xd9 - str 16 | string | 0xda - str 32 | string | 0xdb - array 16 | array | 0xdc - array 32 | array | 0xdd - map 16 | object | 0xde - map 32 | object | 0xdf - negative fixint | number_integer | 0xe0-0xff + positive fixint | number_unsigned | 0x00..0x7F + fixmap | object | 0x80..0x8F + fixarray | array | 0x90..0x9F + fixstr | string | 0xA0..0xBF + nil | `null` | 0xC0 + false | `false` | 0xC2 + true | `true` | 0xC3 + float 32 | number_float | 0xCA + float 64 | number_float | 0xCB + uint 8 | number_unsigned | 0xCC + uint 16 | number_unsigned | 0xCD + uint 32 | number_unsigned | 0xCE + uint 64 | number_unsigned | 0xCF + int 8 | number_integer | 0xD0 + int 16 | number_integer | 0xD1 + int 32 | number_integer | 0xD2 + int 64 | number_integer | 0xD3 + str 8 | string | 0xD9 + str 16 | string | 0xDA + str 32 | string | 0xDB + array 16 | array | 0xDC + array 32 | array | 0xDD + map 16 | object | 0xDE + map 32 | object | 0xDF + negative fixint | number_integer | 0xE0-0xFF @warning The mapping is **incomplete** in the sense that not all MessagePack types can be converted to a JSON value. The following MessagePack types are not supported and will yield parse errors: - - bin 8 - bin 32 (0xc4..0xc6) - - ext 8 - ext 32 (0xc7..0xc9) - - fixext 1 - fixext 16 (0xd4..0xd8) + - bin 8 - bin 32 (0xC4..0xC6) + - ext 8 - ext 32 (0xC7..0xC9) + - fixext 1 - fixext 16 (0xD4..0xD8) @note Any MessagePack output created @ref to_msgpack can be successfully parsed by @ref from_msgpack. @@ -13601,6 +13741,9 @@ class basic_json pointer @a ptr. As `at` provides checked access (and no elements are implicitly inserted), the index '-' is always invalid. See example below. + @throw out_of_range.403 if the JSON pointer describes a key of an object + which cannot be found. See example below. + @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved. See example below. @@ -13641,6 +13784,9 @@ class basic_json pointer @a ptr. As `at` provides checked access (and no elements are implicitly inserted), the index '-' is always invalid. See example below. + @throw out_of_range.403 if the JSON pointer describes a key of an object + which cannot be found. See example below. + @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved. See example below. @@ -13856,7 +14002,7 @@ class basic_json } else { - const auto idx = std::stoi(last_path); + const auto idx = json_pointer::array_index(last_path); if (JSON_UNLIKELY(static_cast<size_type>(idx) > parent.size())) { // avoid undefined behavior @@ -13904,7 +14050,7 @@ class basic_json else if (parent.is_array()) { // note erase performs range check - parent.erase(static_cast<size_type>(std::stoi(last_path))); + parent.erase(static_cast<size_type>(json_pointer::array_index(last_path))); } }; @@ -13999,7 +14145,12 @@ class basic_json const json_pointer from_ptr(from_path); // the "from" location must exist - use at() - result[ptr] = result.at(from_ptr); + basic_json v = result.at(from_ptr); + + // The copy is functionally identical to an "add" + // operation at the target location using the value + // specified in the "from" member. + operation_add(ptr, v); break; } @@ -14141,7 +14292,7 @@ class basic_json case value_t::object: { // first pass: traverse this object's elements - for (auto it = source.begin(); it != source.end(); ++it) + for (auto it = source.cbegin(); it != source.cend(); ++it) { // escape the key name to be used in a JSON patch const auto key = json_pointer::escape(it.key()); @@ -14163,7 +14314,7 @@ class basic_json } // second pass: traverse other object's elements - for (auto it = target.begin(); it != target.end(); ++it) + for (auto it = target.cbegin(); it != target.cend(); ++it) { if (source.find(it.key()) == source.end()) { @@ -14256,7 +14407,7 @@ json_pointer::get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const // create an entry in the array JSON_TRY { - result = &result->operator[](static_cast<size_type>(std::stoi(reference_token))); + result = &result->operator[](static_cast<size_type>(array_index(reference_token))); } JSON_CATCH(std::invalid_argument&) { @@ -14333,7 +14484,7 @@ json_pointer::get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const JSON_TRY { ptr = &ptr->operator[]( - static_cast<size_type>(std::stoi(reference_token))); + static_cast<size_type>(array_index(reference_token))); } JSON_CATCH(std::invalid_argument&) { @@ -14388,7 +14539,7 @@ json_pointer::get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const // note: at performs range check JSON_TRY { - ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token))); + ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); } JSON_CATCH(std::invalid_argument&) { @@ -14443,7 +14594,7 @@ json_pointer::get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const JSON_TRY { ptr = &ptr->operator[]( - static_cast<size_type>(std::stoi(reference_token))); + static_cast<size_type>(array_index(reference_token))); } JSON_CATCH(std::invalid_argument&) { @@ -14497,7 +14648,7 @@ json_pointer::get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const // note: at performs range check JSON_TRY { - ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token))); + ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); } JSON_CATCH(std::invalid_argument&) { |