From bf452cbc2ae2b209ec262ce858deca470d086f24 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Mon, 25 May 2020 15:54:14 +0100 Subject: refactor(3p/nix): Replace tokenizeStrings with absl::StrSplit This function was a custom (and inefficient in the case of single-character delimiters) string splitter which was used all over the codebase. Abseil provides an appropriate replacement function. --- third_party/nix/src/libstore/binary-cache-store.cc | 3 ++- third_party/nix/src/libstore/build.cc | 7 ++++-- third_party/nix/src/libstore/builtins/buildenv.cc | 14 +++++++---- third_party/nix/src/libstore/derivations.cc | 18 +++++++++------ third_party/nix/src/libstore/derivations.hh | 2 +- third_party/nix/src/libstore/download.cc | 7 +++--- third_party/nix/src/libstore/gc.cc | 27 +++------------------- third_party/nix/src/libstore/globals.cc | 18 ++++++--------- third_party/nix/src/libstore/local-store.cc | 10 ++++---- third_party/nix/src/libstore/local-store.hh | 6 ++++- third_party/nix/src/libstore/machines.cc | 20 +++++++++------- .../nix/src/libstore/nar-info-disk-cache.cc | 11 +++++---- third_party/nix/src/libstore/nar-info.cc | 3 ++- third_party/nix/src/libstore/parsed-derivations.cc | 4 +++- third_party/nix/src/libstore/ssh.cc | 6 +++-- third_party/nix/src/libstore/store-api.cc | 4 +++- 16 files changed, 84 insertions(+), 76 deletions(-) (limited to 'third_party/nix/src/libstore') diff --git a/third_party/nix/src/libstore/binary-cache-store.cc b/third_party/nix/src/libstore/binary-cache-store.cc index ea36078435ba..f631c9dee231 100644 --- a/third_party/nix/src/libstore/binary-cache-store.cc +++ b/third_party/nix/src/libstore/binary-cache-store.cc @@ -6,6 +6,7 @@ #include #include +#include #include "archive.hh" #include "compression.hh" @@ -41,7 +42,7 @@ void BinaryCacheStore::init() { upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); } else { - for (auto& line : tokenizeString(*cacheInfo, "\n")) { + for (auto& line : absl::StrSplit(*cacheInfo, absl::ByChar('\n'))) { size_t colon = line.find(':'); if (colon == std::string::npos) { continue; diff --git a/third_party/nix/src/libstore/build.cc b/third_party/nix/src/libstore/build.cc index c0fa0074d6ea..a60f5ef173e6 100644 --- a/third_party/nix/src/libstore/build.cc +++ b/third_party/nix/src/libstore/build.cc @@ -10,10 +10,12 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -1968,7 +1970,7 @@ void DerivationGoal::startBuilder() { by `nix-store --register-validity'. However, the deriver fields are left empty. */ std::string s = get(drv->env, "exportReferencesGraph"); - auto ss = tokenizeString(s); + std::vector ss = absl::StrSplit(s, absl::ByAnyChar(" \t\n\r")); if (ss.size() % 2 != 0) { throw BuildError( format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); @@ -2481,7 +2483,8 @@ void DerivationGoal::initTmpDir() { needed (attributes are not passed through the environment, so there is no size constraint). */ if (!parsedDrv->getStructuredAttrs()) { - auto passAsFile = tokenizeString(get(drv->env, "passAsFile")); + std::set passAsFile = + absl::StrSplit(get(drv->env, "passAsFile"), absl::ByAnyChar(" \t\n\r")); int fileNr = 0; for (auto& i : drv->env) { if (passAsFile.find(i.first) == passAsFile.end()) { diff --git a/third_party/nix/src/libstore/builtins/buildenv.cc b/third_party/nix/src/libstore/builtins/buildenv.cc index 92cf2f8c1eec..db093663bfef 100644 --- a/third_party/nix/src/libstore/builtins/buildenv.cc +++ b/third_party/nix/src/libstore/builtins/buildenv.cc @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -134,12 +135,14 @@ static void addPkg(const Path& pkgDir, int priority) { createLinks(pkgDir, out, priority); try { - for (const auto& p : tokenizeString>( + for (auto p : absl::StrSplit( readFile(pkgDir + "/nix-support/propagated-user-env-packages"), - " \n")) - if (!done.count(p)) { - postponed.insert(p); + absl::ByAnyChar(" \n"))) { + auto pkg = std::string(p); + if (!done.count(pkg)) { + postponed.insert(pkg); } + } } catch (SysError& e) { if (e.errNo != ENOENT && e.errNo != ENOTDIR) { throw; @@ -172,7 +175,8 @@ void builtinBuildenv(const BasicDerivation& drv) { /* Convert the stuff we get from the environment back into a * coherent data type. */ Packages pkgs; - auto derivations = tokenizeString(getAttr("derivations")); + Strings derivations = + absl::StrSplit(getAttr("derivations"), absl::ByAnyChar(" \t\n\r")); while (!derivations.empty()) { /* !!! We're trusting the caller to structure derivations env var correctly */ diff --git a/third_party/nix/src/libstore/derivations.cc b/third_party/nix/src/libstore/derivations.cc index 9f2aa5ea924f..b2590f5e28e6 100644 --- a/third_party/nix/src/libstore/derivations.cc +++ b/third_party/nix/src/libstore/derivations.cc @@ -1,6 +1,8 @@ #include "derivations.hh" #include +#include +#include #include "fs-accessor.hh" #include "globals.hh" @@ -357,13 +359,15 @@ Hash hashDerivationModulo(Store& store, Derivation drv) { return hashString(htSHA256, drv.unparse()); } -DrvPathWithOutputs parseDrvPathWithOutputs(const std::string& s) { - size_t n = s.find('!'); - return n == std::string::npos - ? DrvPathWithOutputs(s, std::set()) - : DrvPathWithOutputs(std::string(s, 0, n), - tokenizeString >( - std::string(s, n + 1), ",")); +// TODO(tazjin): doc comment? +DrvPathWithOutputs parseDrvPathWithOutputs(absl::string_view path) { + auto pos = path.find('!'); + if (pos == absl::string_view::npos) { + return DrvPathWithOutputs(path, std::set()); + } + + return DrvPathWithOutputs(path.substr(pos + 1), + absl::StrSplit(path, absl::ByChar(','))); } Path makeDrvPathWithOutputs(const Path& drvPath, diff --git a/third_party/nix/src/libstore/derivations.hh b/third_party/nix/src/libstore/derivations.hh index ae365e68bf3a..170f09804dee 100644 --- a/third_party/nix/src/libstore/derivations.hh +++ b/third_party/nix/src/libstore/derivations.hh @@ -88,7 +88,7 @@ extern DrvHashes drvHashes; // FIXME: global, not thread-safe (/nix/store/hash-foo!out1,out2,...) into the derivation path and the outputs. */ typedef std::pair > DrvPathWithOutputs; -DrvPathWithOutputs parseDrvPathWithOutputs(const std::string& s); +DrvPathWithOutputs parseDrvPathWithOutputs(absl::string_view path); Path makeDrvPathWithOutputs(const Path& drvPath, const std::set& outputs); diff --git a/third_party/nix/src/libstore/download.cc b/third_party/nix/src/libstore/download.cc index 8b36063dabc5..92a63087289e 100644 --- a/third_party/nix/src/libstore/download.cc +++ b/third_party/nix/src/libstore/download.cc @@ -3,6 +3,7 @@ #include #include #include +#include #include "archive.hh" #include "compression.hh" @@ -180,7 +181,7 @@ struct CurlDownloader : public Downloader { << "': " << absl::StripAsciiWhitespace(line); if (line.compare(0, 5, "HTTP/") == 0) { // new response starts result.etag = ""; - auto ss = tokenizeString>(line, " "); + std::vector ss = absl::StrSplit(line, absl::ByChar(' ')); status = ss.size() >= 2 ? ss[1] : ""; result.data = std::make_shared(); result.bodySize = 0; @@ -894,8 +895,8 @@ CachedDownloadResult Downloader::downloadCached( storePath = readLink(fileLink); store->addTempRoot(storePath); if (store->isValidPath(storePath)) { - auto ss = - tokenizeString>(readFile(dataFile), "\n"); + std::vector ss = + absl::StrSplit(readFile(dataFile), absl::ByChar('\n')); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; if (absl::SimpleAtoi(ss[2], &lastChecked) && diff --git a/third_party/nix/src/libstore/gc.cc b/third_party/nix/src/libstore/gc.cc index 262f17df0338..5c1ea3d58b69 100644 --- a/third_party/nix/src/libstore/gc.cc +++ b/third_party/nix/src/libstore/gc.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -422,8 +423,8 @@ void LocalStore::findRuntimeRoots(Roots& roots, bool censor) { try { auto mapFile = fmt("/proc/%s/maps", ent->d_name); - auto mapLines = tokenizeString>( - readFile(mapFile, true), "\n"); + std::vector mapLines = + absl::StrSplit(readFile(mapFile, true), absl::ByChar('\n')); for (const auto& line : mapLines) { auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) { @@ -452,31 +453,9 @@ void LocalStore::findRuntimeRoots(Roots& roots, bool censor) { } } -#if !defined(__linux__) - // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test - // to fail. See: https://github.com/NixOS/nix/issues/3011 Because of this we - // disable lsof when running the tests. - if (getEnv("_NIX_TEST_NO_LSOF") == "") { - try { - std::regex lsofRegex(R"(^n(/.*)$)"); - auto lsofLines = tokenizeString>( - runProgram(LSOF, true, {"-n", "-w", "-F", "n"}), "\n"); - for (const auto& line : lsofLines) { - std::smatch match; - if (std::regex_match(line, match, lsofRegex)) - unchecked[match[1]].emplace("{lsof}"); - } - } catch (ExecError& e) { - /* lsof not installed, lsof failed */ - } - } -#endif - -#if defined(__linux__) readFileRoots("/proc/sys/kernel/modprobe", unchecked); readFileRoots("/proc/sys/kernel/fbsplash", unchecked); readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); -#endif for (auto& [target, links] : unchecked) { if (isInStore(target)) { diff --git a/third_party/nix/src/libstore/globals.cc b/third_party/nix/src/libstore/globals.cc index 7307f035275f..f218534cc9f8 100644 --- a/third_party/nix/src/libstore/globals.cc +++ b/third_party/nix/src/libstore/globals.cc @@ -5,6 +5,8 @@ #include #include +#include +#include #include #include "archive.hh" @@ -20,9 +22,6 @@ namespace nix { must be deleted and recreated on startup.) */ #define DEFAULT_SOCKET_PATH "/daemon-socket/socket" -// TODO(tazjin): this was __APPLE__ specific, still needed? -#define DEFAULT_ALLOWED_IMPURE_PREFIXES "" - Settings settings; static GlobalConfig::Register r1(&settings); @@ -55,21 +54,18 @@ Settings::Settings() } /* Backwards compatibility. */ + // TODO(tazjin): still? auto s = getEnv("NIX_REMOTE_SYSTEMS"); if (!s.empty()) { Strings ss; - for (auto& p : tokenizeString(s, ":")) { - ss.push_back("@" + p); + for (auto p : absl::StrSplit(s, absl::ByChar(':'))) { + ss.push_back(absl::StrCat("@", p)); } builders = concatStringsSep(" ", ss); } -#if defined(__linux__) && defined(SANDBOX_SHELL) - sandboxPaths = tokenizeString("/bin/sh=" SANDBOX_SHELL); -#endif - - allowedImpureHostPrefixes = - tokenizeString(DEFAULT_ALLOWED_IMPURE_PREFIXES); + sandboxPaths = + absl::StrSplit("/bin/sh=" SANDBOX_SHELL, absl::ByAnyChar(" \t\n\r")); } void loadConfFile() { diff --git a/third_party/nix/src/libstore/local-store.cc b/third_party/nix/src/libstore/local-store.cc index faea0fbce56a..ebc912969423 100644 --- a/third_party/nix/src/libstore/local-store.cc +++ b/third_party/nix/src/libstore/local-store.cc @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -477,14 +478,15 @@ static void canonicalisePathMetaData_(const Path& path, uid_t fromUid, throw SysError("querying extended attributes of '%s'", path); } - for (auto& eaName : tokenizeString( - std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { + for (auto& eaName : + absl::StrSplit(std::string(eaBuf.data(), eaSize), + absl::ByString(std::string("\000", 1)))) { /* Ignore SELinux security labels since these cannot be removed even by root. */ if (eaName == "security.selinux") { continue; } - if (lremovexattr(path.c_str(), eaName.c_str()) == -1) { + if (lremovexattr(path.c_str(), std::string(eaName).c_str()) == -1) { throw SysError("removing extended attribute '%s' from '%s'", eaName, path); } @@ -702,7 +704,7 @@ void LocalStore::queryPathInfoUncached( s = (const char*)sqlite3_column_text(state->stmtQueryPathInfo, 6); if (s != nullptr) { - info->sigs = tokenizeString(s, " "); + info->sigs = absl::StrSplit(s, absl::ByChar(' ')); } s = (const char*)sqlite3_column_text(state->stmtQueryPathInfo, 7); diff --git a/third_party/nix/src/libstore/local-store.hh b/third_party/nix/src/libstore/local-store.hh index c1bfc0876412..178cadf92bf6 100644 --- a/third_party/nix/src/libstore/local-store.hh +++ b/third_party/nix/src/libstore/local-store.hh @@ -5,6 +5,8 @@ #include #include +#include + #include "pathlocks.hh" #include "sqlite.hh" #include "store-api.hh" @@ -94,7 +96,9 @@ class LocalStore : public LocalFSStore { public: // Hack for build-remote.cc. - PathSet locksHeld = tokenizeString(getEnv("NIX_HELD_LOCKS")); + // TODO(tazjin): remove this when we've got gRPC + PathSet locksHeld = + absl::StrSplit(getEnv("NIX_HELD_LOCKS"), absl::ByAnyChar(" \t\n\r")); /* Initialise the local store, upgrading the schema if necessary. */ diff --git a/third_party/nix/src/libstore/machines.cc b/third_party/nix/src/libstore/machines.cc index bbb84784c4f0..9ad20a25f8f0 100644 --- a/third_party/nix/src/libstore/machines.cc +++ b/third_party/nix/src/libstore/machines.cc @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -51,14 +52,15 @@ bool Machine::mandatoryMet(const std::set& features) const { } void parseMachines(const std::string& s, Machines& machines) { - for (auto line : tokenizeString>(s, "\n;")) { - line.erase(std::find(line.begin(), line.end(), '#'), line.end()); - if (line.empty()) { + for (auto line : absl::StrSplit(s, absl::ByAnyChar("\n;"))) { + // Skip empty lines & comments + line = absl::StripAsciiWhitespace(line); + if (line.empty() || line[line.find_first_not_of(" \t")] == '#') { continue; } if (line[0] == '@') { - auto file = absl::StripAsciiWhitespace(std::string(line, 1)); + auto file = absl::StripAsciiWhitespace(line.substr(1)); try { parseMachines(readFile(file), machines); } catch (const SysError& e) { @@ -70,7 +72,8 @@ void parseMachines(const std::string& s, Machines& machines) { continue; } - auto tokens = tokenizeString>(line); + std::vector tokens = + absl::StrSplit(line, absl::ByAnyChar(" \t\n\r")); auto sz = tokens.size(); if (sz < 1) { throw FormatError("bad machine specification '%s'", line); @@ -80,15 +83,16 @@ void parseMachines(const std::string& s, Machines& machines) { return tokens.size() > n && !tokens[n].empty() && tokens[n] != "-"; }; + // TODO(tazjin): what??? machines.emplace_back( tokens[0], - isSet(1) ? tokenizeString>(tokens[1], ",") + isSet(1) ? absl::StrSplit(tokens[1], absl::ByChar(',')) : std::vector{settings.thisSystem}, isSet(2) ? tokens[2] : "", isSet(3) ? std::stoull(tokens[3]) : 1LL, isSet(4) ? std::stoull(tokens[4]) : 1LL, - isSet(5) ? tokenizeString>(tokens[5], ",") + isSet(5) ? absl::StrSplit(tokens[5], absl::ByChar(',')) : std::set{}, - isSet(6) ? tokenizeString>(tokens[6], ",") + isSet(6) ? absl::StrSplit(tokens[6], absl::ByChar(',')) : std::set{}, isSet(7) ? tokens[7] : ""); } diff --git a/third_party/nix/src/libstore/nar-info-disk-cache.cc b/third_party/nix/src/libstore/nar-info-disk-cache.cc index 19ef2c9d7461..0a7a0dc22c51 100644 --- a/third_party/nix/src/libstore/nar-info-disk-cache.cc +++ b/third_party/nix/src/libstore/nar-info-disk-cache.cc @@ -1,5 +1,7 @@ #include "nar-info-disk-cache.hh" +#include +#include #include #include @@ -227,14 +229,15 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache { narInfo->fileSize = queryNAR.getInt(5); narInfo->narHash = Hash(queryNAR.getStr(6)); narInfo->narSize = queryNAR.getInt(7); - for (auto& r : tokenizeString(queryNAR.getStr(8), " ")) { - narInfo->references.insert(cache.storeDir + "/" + r); + for (auto r : absl::StrSplit(queryNAR.getStr(8), absl::ByChar(' '))) { + narInfo->references.insert(absl::StrCat(cache.storeDir, "/", r)); } if (!queryNAR.isNull(9)) { narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9); } - for (auto& sig : tokenizeString(queryNAR.getStr(10), " ")) { - narInfo->sigs.insert(sig); + for (auto& sig : + absl::StrSplit(queryNAR.getStr(10), absl::ByChar(' '))) { + narInfo->sigs.insert(std::string(sig)); } narInfo->ca = queryNAR.getStr(11); diff --git a/third_party/nix/src/libstore/nar-info.cc b/third_party/nix/src/libstore/nar-info.cc index 9f4c53e19b59..1dc0b54cf1e6 100644 --- a/third_party/nix/src/libstore/nar-info.cc +++ b/third_party/nix/src/libstore/nar-info.cc @@ -1,6 +1,7 @@ #include "nar-info.hh" #include +#include #include "globals.hh" @@ -59,7 +60,7 @@ NarInfo::NarInfo(const Store& store, const std::string& s, corrupt(); } } else if (name == "References") { - auto refs = tokenizeString(value, " "); + std::vector refs = absl::StrSplit(value, absl::ByChar(' ')); if (!references.empty()) { corrupt(); } diff --git a/third_party/nix/src/libstore/parsed-derivations.cc b/third_party/nix/src/libstore/parsed-derivations.cc index 1186203249de..571f49ad59e2 100644 --- a/third_party/nix/src/libstore/parsed-derivations.cc +++ b/third_party/nix/src/libstore/parsed-derivations.cc @@ -1,5 +1,7 @@ #include "parsed-derivations.hh" +#include + namespace nix { ParsedDerivation::ParsedDerivation(const Path& drvPath, BasicDerivation& drv) @@ -86,7 +88,7 @@ std::optional ParsedDerivation::getStringsAttr( if (i == drv.env.end()) { return {}; } - return tokenizeString(i->second); + return absl::StrSplit(i->second, absl::ByAnyChar(" \t\n\r")); } } diff --git a/third_party/nix/src/libstore/ssh.cc b/third_party/nix/src/libstore/ssh.cc index 76c78b2fcd77..1b09eb42c6b9 100644 --- a/third_party/nix/src/libstore/ssh.cc +++ b/third_party/nix/src/libstore/ssh.cc @@ -3,6 +3,7 @@ #include #include +#include namespace nix { @@ -20,8 +21,9 @@ SSHMaster::SSHMaster(const std::string& host, std::string keyFile, } void SSHMaster::addCommonSSHOpts(Strings& args) { - for (auto& i : tokenizeString(getEnv("NIX_SSHOPTS"))) { - args.push_back(i); + for (auto& i : + absl::StrSplit(getEnv("NIX_SSHOPTS"), absl::ByAnyChar(" \t\n\r"))) { + args.push_back(std::string(i)); } if (!keyFile.empty()) { args.insert(args.end(), {"-i", keyFile}); diff --git a/third_party/nix/src/libstore/store-api.cc b/third_party/nix/src/libstore/store-api.cc index 9440d78d5f12..afdbc14e29a3 100644 --- a/third_party/nix/src/libstore/store-api.cc +++ b/third_party/nix/src/libstore/store-api.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include "crypto.hh" @@ -858,7 +859,8 @@ std::pair splitUriAndParams( Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { - for (const auto& s : tokenizeString(uri.substr(q + 1), "&")) { + Strings parts = absl::StrSplit(uri.substr(q + 1), absl::ByChar('&')); + for (const auto& s : parts) { auto e = s.find('='); if (e != std::string::npos) { auto value = s.substr(e + 1); -- cgit 1.4.1