diff options
Diffstat (limited to 'src/libutil')
32 files changed, 1769 insertions, 2070 deletions
diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc index 3e21f43a2e9d..3cbdf878617a 100644 --- a/src/libutil/affinity.cc +++ b/src/libutil/affinity.cc @@ -2,14 +2,14 @@ #include "util.hh" #include "affinity.hh" -#if HAVE_SCHED_H +#if __linux__ #include <sched.h> #endif namespace nix { -#if HAVE_SCHED_SETAFFINITY +#if __linux__ static bool didSaveAffinity = false; static cpu_set_t savedAffinity; #endif @@ -17,7 +17,7 @@ static cpu_set_t savedAffinity; void setAffinityTo(int cpu) { -#if HAVE_SCHED_SETAFFINITY +#if __linux__ if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; didSaveAffinity = true; printMsg(lvlDebug, format("locking this thread to CPU %1%") % cpu); @@ -32,7 +32,7 @@ void setAffinityTo(int cpu) int lockToCurrentCPU() { -#if HAVE_SCHED_SETAFFINITY +#if __linux__ int cpu = sched_getcpu(); if (cpu != -1) setAffinityTo(cpu); return cpu; @@ -44,7 +44,7 @@ int lockToCurrentCPU() void restoreAffinity() { -#if HAVE_SCHED_SETAFFINITY +#if __linux__ if (!didSaveAffinity) return; if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) printMsg(lvlError, "failed to restore affinity %1%"); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 9e16e04ae4b5..5363496c272e 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -29,7 +29,7 @@ bool useCaseHack = false; #endif -static string archiveVersion1 = "nix-archive-1"; +const std::string narVersionMagic1 = "nix-archive-1"; static string caseHackSuffix = "~nix~case~hack~"; @@ -39,8 +39,7 @@ PathFilter defaultPathFilter; static void dumpContents(const Path & path, size_t size, Sink & sink) { - writeString("contents", sink); - writeLongLong(size, sink); + sink << "contents" << size; AutoCloseFD fd = open(path.c_str(), O_RDONLY); if (fd == -1) throw SysError(format("opening file ‘%1%’") % path); @@ -65,21 +64,17 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) if (lstat(path.c_str(), &st)) throw SysError(format("getting attributes of path ‘%1%’") % path); - writeString("(", sink); + sink << "("; if (S_ISREG(st.st_mode)) { - writeString("type", sink); - writeString("regular", sink); - if (st.st_mode & S_IXUSR) { - writeString("executable", sink); - writeString("", sink); - } + sink << "type" << "regular"; + if (st.st_mode & S_IXUSR) + sink << "executable" << ""; dumpContents(path, (size_t) st.st_size, sink); } else if (S_ISDIR(st.st_mode)) { - writeString("type", sink); - writeString("directory", sink); + sink << "type" << "directory"; /* If we're on a case-insensitive system like Mac OS X, undo the case hack applied by restorePath(). */ @@ -101,36 +96,34 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) for (auto & i : unhacked) if (filter(path + "/" + i.first)) { - writeString("entry", sink); - writeString("(", sink); - writeString("name", sink); - writeString(i.first, sink); - writeString("node", sink); + sink << "entry" << "(" << "name" << i.first << "node"; dump(path + "/" + i.second, sink, filter); - writeString(")", sink); + sink << ")"; } } - else if (S_ISLNK(st.st_mode)) { - writeString("type", sink); - writeString("symlink", sink); - writeString("target", sink); - writeString(readLink(path), sink); - } + else if (S_ISLNK(st.st_mode)) + sink << "type" << "symlink" << "target" << readLink(path); else throw Error(format("file ‘%1%’ has an unsupported type") % path); - writeString(")", sink); + sink << ")"; } void dumpPath(const Path & path, Sink & sink, PathFilter & filter) { - writeString(archiveVersion1, sink); + sink << narVersionMagic1; dump(path, sink, filter); } +void dumpString(const std::string & s, Sink & sink) +{ + sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")"; +} + + static SerialisationError badArchive(string s) { return SerialisationError("bad archive: " + s); @@ -227,7 +220,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path) } else if (s == "executable" && type == tpRegular) { - readString(source); + auto s = readString(source); + if (s != "") throw badArchive("executable marker has non-empty value"); sink.isExecutable(); } @@ -256,7 +250,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) if (i != names.end()) { printMsg(lvlDebug, format("case collision between ‘%1%’ and ‘%2%’") % i->first % name); name += caseHackSuffix; - name += int2String(++i->second); + name += std::to_string(++i->second); } else names[name] = 0; } @@ -288,7 +282,7 @@ void parseDump(ParseSink & sink, Source & source) /* This generally means the integer at the start couldn't be decoded. Ignore and throw the exception below. */ } - if (version != archiveVersion1) + if (version != narVersionMagic1) throw badArchive("input doesn't look like a Nix archive"); parse(sink, source, ""); } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index c216e9768fd1..d58b91df0461 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -55,6 +55,9 @@ extern PathFilter defaultPathFilter; void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +void dumpString(const std::string & s, Sink & sink); + +/* FIXME: fix this API, it sucks. */ struct ParseSink { virtual void createDirectory(const Path & path) { }; @@ -76,4 +79,7 @@ void restorePath(const Path & path, Source & source); extern bool useCaseHack; +extern const std::string narVersionMagic1; + + } diff --git a/src/libutil/args.cc b/src/libutil/args.cc new file mode 100644 index 000000000000..115484f9e6c7 --- /dev/null +++ b/src/libutil/args.cc @@ -0,0 +1,180 @@ +#include "args.hh" +#include "hash.hh" + +namespace nix { + +void Args::parseCmdline(const Strings & _cmdline) +{ + Strings pendingArgs; + bool dashDash = false; + + Strings cmdline(_cmdline); + + for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { + + auto arg = *pos; + + /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f', + `-j3` -> `-j 3`). */ + if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) { + *pos = (string) "-" + arg[1]; + auto next = pos; ++next; + for (unsigned int j = 2; j < arg.length(); j++) + if (isalpha(arg[j])) + cmdline.insert(next, (string) "-" + arg[j]); + else { + cmdline.insert(next, string(arg, j)); + break; + } + arg = *pos; + } + + if (!dashDash && arg == "--") { + dashDash = true; + ++pos; + } + else if (!dashDash && std::string(arg, 0, 1) == "-") { + if (!processFlag(pos, cmdline.end())) + throw UsageError(format("unrecognised flag ‘%1%’") % arg); + } + else { + pendingArgs.push_back(*pos++); + if (processArgs(pendingArgs, false)) + pendingArgs.clear(); + } + } + + processArgs(pendingArgs, true); +} + +void Args::printHelp(const string & programName, std::ostream & out) +{ + std::cout << "Usage: " << programName << " <FLAGS>..."; + for (auto & exp : expectedArgs) { + std::cout << renderLabels({exp.label}); + // FIXME: handle arity > 1 + if (exp.arity == 0) std::cout << "..."; + } + std::cout << "\n"; + + auto s = description(); + if (s != "") + std::cout << "\nSummary: " << s << ".\n"; + + if (longFlags.size()) { + std::cout << "\n"; + std::cout << "Flags:\n"; + printFlags(out); + } +} + +void Args::printFlags(std::ostream & out) +{ + Table2 table; + for (auto & flag : longFlags) + table.push_back(std::make_pair( + (flag.second.shortName ? std::string("-") + flag.second.shortName + ", " : " ") + + "--" + flag.first + renderLabels(flag.second.labels), + flag.second.description)); + printTable(out, table); +} + +bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + assert(pos != end); + + auto process = [&](const std::string & name, const Flag & flag) -> bool { + ++pos; + Strings args; + for (size_t n = 0 ; n < flag.arity; ++n) { + if (pos == end) + throw UsageError(format("flag ‘%1%’ requires %2% argument(s)") + % name % flag.arity); + args.push_back(*pos++); + } + flag.handler(args); + return true; + }; + + if (string(*pos, 0, 2) == "--") { + auto i = longFlags.find(string(*pos, 2)); + if (i == longFlags.end()) return false; + return process("--" + i->first, i->second); + } + + if (string(*pos, 0, 1) == "-" && pos->size() == 2) { + auto c = (*pos)[1]; + auto i = shortFlags.find(c); + if (i == shortFlags.end()) return false; + return process(std::string("-") + c, i->second); + } + + return false; +} + +bool Args::processArgs(const Strings & args, bool finish) +{ + if (expectedArgs.empty()) { + if (!args.empty()) + throw UsageError(format("unexpected argument ‘%1%’") % args.front()); + return true; + } + + auto & exp = expectedArgs.front(); + + bool res = false; + + if ((exp.arity == 0 && finish) || + (exp.arity > 0 && args.size() == exp.arity)) + { + exp.handler(args); + expectedArgs.pop_front(); + res = true; + } + + if (finish && !expectedArgs.empty()) + throw UsageError("more arguments are required"); + + return res; +} + +void Args::mkHashTypeFlag(const std::string & name, HashType * ht) +{ + mkFlag1(0, name, "TYPE", "hash algorithm (‘md5’, ‘sha1’, ‘sha256’, or ‘sha512’)", [=](std::string s) { + *ht = parseHashType(s); + if (*ht == htUnknown) + throw UsageError(format("unknown hash type ‘%1%’") % s); + }); +} + +Strings argvToStrings(int argc, char * * argv) +{ + Strings args; + argc--; argv++; + while (argc--) args.push_back(*argv++); + return args; +} + +std::string renderLabels(const Strings & labels) +{ + std::string res; + for (auto label : labels) { + for (auto & c : label) c = std::toupper(c); + res += " <" + label + ">"; + } + return res; +} + +void printTable(std::ostream & out, const Table2 & table) +{ + size_t max = 0; + for (auto & row : table) + max = std::max(max, row.first.size()); + for (auto & row : table) { + out << " " << row.first + << std::string(max - row.first.size() + 2, ' ') + << row.second << "\n"; + } +} + +} diff --git a/src/libutil/args.hh b/src/libutil/args.hh new file mode 100644 index 000000000000..6aa08aacac9e --- /dev/null +++ b/src/libutil/args.hh @@ -0,0 +1,163 @@ +#pragma once + +#include <iostream> +#include <map> +#include <memory> + +#include "util.hh" + +namespace nix { + +MakeError(UsageError, nix::Error); + +enum HashType : char; + +class Args +{ +public: + + /* Parse the command line, throwing a UsageError if something goes + wrong. */ + void parseCmdline(const Strings & cmdline); + + virtual void printHelp(const string & programName, std::ostream & out); + + virtual std::string description() { return ""; } + +protected: + + /* Flags. */ + struct Flag + { + char shortName; + std::string description; + Strings labels; + size_t arity; + std::function<void(Strings)> handler; + }; + + std::map<std::string, Flag> longFlags; + std::map<char, Flag> shortFlags; + + virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); + + void printFlags(std::ostream & out); + + /* Positional arguments. */ + struct ExpectedArg + { + std::string label; + size_t arity; // 0 = any + std::function<void(Strings)> handler; + }; + + std::list<ExpectedArg> expectedArgs; + + virtual bool processArgs(const Strings & args, bool finish); + +public: + + /* Helper functions for constructing flags / positional + arguments. */ + + void mkFlag(char shortName, const std::string & longName, + const Strings & labels, const std::string & description, + size_t arity, std::function<void(Strings)> handler) + { + auto flag = Flag{shortName, description, labels, arity, handler}; + if (shortName) shortFlags[shortName] = flag; + longFlags[longName] = flag; + } + + void mkFlag(char shortName, const std::string & longName, + const std::string & description, std::function<void()> fun) + { + mkFlag(shortName, longName, {}, description, 0, std::bind(fun)); + } + + void mkFlag1(char shortName, const std::string & longName, + const std::string & label, const std::string & description, + std::function<void(std::string)> fun) + { + mkFlag(shortName, longName, {label}, description, 1, [=](Strings ss) { + fun(ss.front()); + }); + } + + void mkFlag(char shortName, const std::string & name, + const std::string & description, bool * dest) + { + mkFlag(shortName, name, description, dest, true); + } + + void mkFlag(char shortName, const std::string & longName, + const std::string & label, const std::string & description, + string * dest) + { + mkFlag1(shortName, longName, label, description, [=](std::string s) { + *dest = s; + }); + } + + void mkHashTypeFlag(const std::string & name, HashType * ht); + + template<class T> + void mkFlag(char shortName, const std::string & longName, const std::string & description, + T * dest, const T & value) + { + mkFlag(shortName, longName, {}, description, 0, [=](Strings ss) { + *dest = value; + }); + } + + template<class I> + void mkIntFlag(char shortName, const std::string & longName, + const std::string & description, I * dest) + { + mkFlag<I>(shortName, longName, description, [=](I n) { + *dest = n; + }); + } + + template<class I> + void mkFlag(char shortName, const std::string & longName, + const std::string & description, std::function<void(I)> fun) + { + mkFlag(shortName, longName, {"N"}, description, 1, [=](Strings ss) { + I n; + if (!string2Int(ss.front(), n)) + throw UsageError(format("flag ‘--%1%’ requires a integer argument") % longName); + fun(n); + }); + } + + /* Expect a string argument. */ + void expectArg(const std::string & label, string * dest) + { + expectedArgs.push_back(ExpectedArg{label, 1, [=](Strings ss) { + *dest = ss.front(); + }}); + } + + /* Expect 0 or more arguments. */ + void expectArgs(const std::string & label, Strings * dest) + { + expectedArgs.push_back(ExpectedArg{label, 0, [=](Strings ss) { + *dest = ss; + }}); + } + + friend class MultiCommand; +}; + +Strings argvToStrings(int argc, char * * argv); + +/* Helper function for rendering argument labels. */ +std::string renderLabels(const Strings & labels); + +/* Helper function for printing 2-column tables. */ +typedef std::vector<std::pair<std::string, std::string>> Table2; + +void printTable(std::ostream & out, const Table2 & table); + +} diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc new file mode 100644 index 000000000000..a3bbb5170d9f --- /dev/null +++ b/src/libutil/compression.cc @@ -0,0 +1,276 @@ +#include "compression.hh" +#include "util.hh" +#include "finally.hh" + +#include <lzma.h> +#include <bzlib.h> +#include <cstdio> +#include <cstring> + +#include <iostream> + +namespace nix { + +static ref<std::string> decompressXZ(const std::string & in) +{ + lzma_stream strm(LZMA_STREAM_INIT); + + lzma_ret ret = lzma_stream_decoder( + &strm, UINT64_MAX, LZMA_CONCATENATED); + if (ret != LZMA_OK) + throw Error("unable to initialise lzma decoder"); + + Finally free([&]() { lzma_end(&strm); }); + + lzma_action action = LZMA_RUN; + uint8_t outbuf[BUFSIZ]; + ref<std::string> res = make_ref<std::string>(); + strm.next_in = (uint8_t *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + + while (true) { + checkInterrupt(); + + if (strm.avail_in == 0) + action = LZMA_FINISH; + + lzma_ret ret = lzma_code(&strm, action); + + if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { + res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == LZMA_STREAM_END) + return res; + + if (ret != LZMA_OK) + throw Error("error while decompressing xz file"); + } +} + +static ref<std::string> decompressBzip2(const std::string & in) +{ + bz_stream strm; + memset(&strm, 0, sizeof(strm)); + + int ret = BZ2_bzDecompressInit(&strm, 0, 0); + if (ret != BZ_OK) + throw Error("unable to initialise bzip2 decoder"); + + Finally free([&]() { BZ2_bzDecompressEnd(&strm); }); + + char outbuf[BUFSIZ]; + ref<std::string> res = make_ref<std::string>(); + strm.next_in = (char *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + + while (true) { + checkInterrupt(); + + int ret = BZ2_bzDecompress(&strm); + + if (strm.avail_out == 0 || ret == BZ_STREAM_END) { + res->append(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == BZ_STREAM_END) + return res; + + if (ret != BZ_OK) + throw Error("error while decompressing bzip2 file"); + } +} + +ref<std::string> compress(const std::string & method, const std::string & in) +{ + StringSink ssink; + auto sink = makeCompressionSink(method, ssink); + (*sink)(in); + sink->finish(); + return ssink.s; +} + +ref<std::string> decompress(const std::string & method, const std::string & in) +{ + if (method == "none") + return make_ref<std::string>(in); + else if (method == "xz") + return decompressXZ(in); + else if (method == "bzip2") + return decompressBzip2(in); + else + throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); +} + +struct NoneSink : CompressionSink +{ + Sink & nextSink; + NoneSink(Sink & nextSink) : nextSink(nextSink) { } + void finish() override { flush(); } + void write(const unsigned char * data, size_t len) override { nextSink(data, len); } +}; + +struct XzSink : CompressionSink +{ + Sink & nextSink; + uint8_t outbuf[BUFSIZ]; + lzma_stream strm = LZMA_STREAM_INIT; + bool finished = false; + + XzSink(Sink & nextSink) : nextSink(nextSink) + { + lzma_ret ret = lzma_easy_encoder( + &strm, 6, LZMA_CHECK_CRC64); + if (ret != LZMA_OK) + throw Error("unable to initialise lzma encoder"); + // FIXME: apply the x86 BCJ filter? + + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + ~XzSink() + { + assert(finished); + lzma_end(&strm); + } + + void finish() override + { + CompressionSink::flush(); + + assert(!finished); + finished = true; + + while (true) { + checkInterrupt(); + + lzma_ret ret = lzma_code(&strm, LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + throw Error("error while flushing xz file"); + + if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == LZMA_STREAM_END) break; + } + } + + void write(const unsigned char * data, size_t len) override + { + assert(!finished); + + strm.next_in = data; + strm.avail_in = len; + + while (strm.avail_in) { + checkInterrupt(); + + lzma_ret ret = lzma_code(&strm, LZMA_RUN); + if (ret != LZMA_OK) + throw Error("error while compressing xz file"); + + if (strm.avail_out == 0) { + nextSink(outbuf, sizeof(outbuf)); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BzipSink : CompressionSink +{ + Sink & nextSink; + char outbuf[BUFSIZ]; + bz_stream strm; + bool finished = false; + + BzipSink(Sink & nextSink) : nextSink(nextSink) + { + memset(&strm, 0, sizeof(strm)); + int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); + if (ret != BZ_OK) + throw Error("unable to initialise bzip2 encoder"); + + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + ~BzipSink() + { + assert(finished); + BZ2_bzCompressEnd(&strm); + } + + void finish() override + { + flush(); + + assert(!finished); + finished = true; + + while (true) { + checkInterrupt(); + + int ret = BZ2_bzCompress(&strm, BZ_FINISH); + if (ret != BZ_FINISH_OK && ret != BZ_STREAM_END) + throw Error("error while flushing bzip2 file"); + + if (strm.avail_out == 0 || ret == BZ_STREAM_END) { + nextSink((unsigned char *) outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == BZ_STREAM_END) break; + } + } + + void write(const unsigned char * data, size_t len) override + { + assert(!finished); + + strm.next_in = (char *) data; + strm.avail_in = len; + + while (strm.avail_in) { + checkInterrupt(); + + int ret = BZ2_bzCompress(&strm, BZ_RUN); + if (ret != BZ_OK) + Error("error while compressing bzip2 file"); + + if (strm.avail_out == 0) { + nextSink((unsigned char *) outbuf, sizeof(outbuf)); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink) +{ + if (method == "none") + return make_ref<NoneSink>(nextSink); + else if (method == "xz") + return make_ref<XzSink>(nextSink); + else if (method == "bzip2") + return make_ref<BzipSink>(nextSink); + else + throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); +} + +} diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh new file mode 100644 index 000000000000..eacf559d65e9 --- /dev/null +++ b/src/libutil/compression.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "ref.hh" +#include "types.hh" +#include "serialise.hh" + +#include <string> + +namespace nix { + +ref<std::string> compress(const std::string & method, const std::string & in); + +ref<std::string> decompress(const std::string & method, const std::string & in); + +struct CompressionSink : BufferedSink +{ + virtual void finish() = 0; +}; + +ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink); + +MakeError(UnknownCompressionMethod, Error); + +} diff --git a/src/libutil/finally.hh b/src/libutil/finally.hh new file mode 100644 index 000000000000..47c64deaecea --- /dev/null +++ b/src/libutil/finally.hh @@ -0,0 +1,12 @@ +#pragma once + +/* A trivial class to run a function at the end of a scope. */ +class Finally +{ +private: + std::function<void()> fun; + +public: + Finally(std::function<void()> fun) : fun(fun) { } + ~Finally() { fun(); } +}; diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index a83ba0a81817..c17f1c4d5150 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -3,16 +3,8 @@ #include <iostream> #include <cstring> -#ifdef HAVE_OPENSSL #include <openssl/md5.h> #include <openssl/sha.h> -#else -extern "C" { -#include "md5.h" -#include "sha1.h" -#include "sha256.h" -} -#endif #include "hash.hh" #include "archive.hh" @@ -40,7 +32,8 @@ Hash::Hash(HashType type) if (type == htMD5) hashSize = md5HashSize; else if (type == htSHA1) hashSize = sha1HashSize; else if (type == htSHA256) hashSize = sha256HashSize; - else throw Error("unknown hash type"); + else if (type == htSHA512) hashSize = sha512HashSize; + else abort(); assert(hashSize <= maxHashSize); memset(hash, 0, maxHashSize); } @@ -71,6 +64,12 @@ bool Hash::operator < (const Hash & h) const } +std::string Hash::to_string(bool base32) const +{ + return printHashType(type) + ":" + (base32 ? printHash32(*this) : printHash(*this)); +} + + const string base16Chars = "0123456789abcdef"; @@ -85,15 +84,28 @@ string printHash(const Hash & hash) } +Hash parseHash(const string & s) +{ + string::size_type colon = s.find(':'); + if (colon == string::npos) + throw BadHash(format("invalid hash ‘%s’") % s); + string hts = string(s, 0, colon); + HashType ht = parseHashType(hts); + if (ht == htUnknown) + throw BadHash(format("unknown hash type ‘%s’") % hts); + return parseHash16or32(ht, string(s, colon + 1)); +} + + Hash parseHash(HashType ht, const string & s) { Hash hash(ht); if (s.length() != hash.hashSize * 2) - throw Error(format("invalid hash ‘%1%’") % s); + throw BadHash(format("invalid hash ‘%1%’") % s); for (unsigned int i = 0; i < hash.hashSize; i++) { string s2(s, i * 2, 2); if (!isxdigit(s2[0]) || !isxdigit(s2[1])) - throw Error(format("invalid hash ‘%1%’") % s); + throw BadHash(format("invalid hash ‘%1%’") % s); std::istringstream str(s2); int n; str >> std::hex >> n; @@ -103,20 +115,14 @@ Hash parseHash(HashType ht, const string & s) } -unsigned int hashLength32(const Hash & hash) -{ - return (hash.hashSize * 8 - 1) / 5 + 1; -} - - // omitted: E O U T const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; string printHash32(const Hash & hash) { - Hash hash2(hash); - unsigned int len = hashLength32(hash); + size_t len = hash.base32Len(); + assert(len); string s; s.reserve(len); @@ -144,7 +150,7 @@ string printHash16or32(const Hash & hash) Hash parseHash32(HashType ht, const string & s) { Hash hash(ht); - unsigned int len = hashLength32(ht); + size_t len = hash.base32Len(); assert(s.size() == len); for (unsigned int n = 0; n < len; ++n) { @@ -153,7 +159,7 @@ Hash parseHash32(HashType ht, const string & s) for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ if (base32Chars[digit] == c) break; if (digit >= 32) - throw Error(format("invalid base-32 hash ‘%1%’") % s); + throw BadHash(format("invalid base-32 hash ‘%1%’") % s); unsigned int b = n * 5; unsigned int i = b / 8; unsigned int j = b % 8; @@ -171,11 +177,11 @@ Hash parseHash16or32(HashType ht, const string & s) if (s.size() == hash.hashSize * 2) /* hexadecimal representation */ hash = parseHash(ht, s); - else if (s.size() == hashLength32(hash)) + else if (s.size() == hash.base32Len()) /* base-32 representation */ hash = parseHash32(ht, s); else - throw Error(format("hash ‘%1%’ has wrong length for hash type ‘%2%’") + throw BadHash(format("hash ‘%1%’ has wrong length for hash type ‘%2%’") % s % printHashType(ht)); return hash; } @@ -199,6 +205,7 @@ union Ctx MD5_CTX md5; SHA_CTX sha1; SHA256_CTX sha256; + SHA512_CTX sha512; }; @@ -207,6 +214,7 @@ static void start(HashType ht, Ctx & ctx) if (ht == htMD5) MD5_Init(&ctx.md5); else if (ht == htSHA1) SHA1_Init(&ctx.sha1); else if (ht == htSHA256) SHA256_Init(&ctx.sha256); + else if (ht == htSHA512) SHA512_Init(&ctx.sha512); } @@ -216,6 +224,7 @@ static void update(HashType ht, Ctx & ctx, if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len); + else if (ht == htSHA512) SHA512_Update(&ctx.sha512, bytes, len); } @@ -224,6 +233,7 @@ static void finish(HashType ht, Ctx & ctx, unsigned char * hash) if (ht == htMD5) MD5_Final(hash, &ctx.md5); else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1); else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256); + else if (ht == htSHA512) SHA512_Final(hash, &ctx.sha512); } @@ -321,6 +331,7 @@ HashType parseHashType(const string & s) if (s == "md5") return htMD5; else if (s == "sha1") return htSHA1; else if (s == "sha256") return htSHA256; + else if (s == "sha512") return htSHA512; else return htUnknown; } @@ -330,7 +341,8 @@ string printHashType(HashType ht) if (ht == htMD5) return "md5"; else if (ht == htSHA1) return "sha1"; else if (ht == htSHA256) return "sha256"; - else throw Error("cannot print unknown hash type"); + else if (ht == htSHA512) return "sha512"; + else abort(); } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 2c6f176ec74c..02e213fc7b35 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -7,30 +7,37 @@ namespace nix { -typedef enum { htUnknown, htMD5, htSHA1, htSHA256 } HashType; +MakeError(BadHash, Error); + + +enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 }; const int md5HashSize = 16; const int sha1HashSize = 20; const int sha256HashSize = 32; +const int sha512HashSize = 64; extern const string base32Chars; struct Hash { - static const unsigned int maxHashSize = 32; + static const unsigned int maxHashSize = 64; unsigned int hashSize; unsigned char hash[maxHashSize]; HashType type; - /* Create an unusable hash object. */ + /* Create an unset hash object. */ Hash(); /* Create a zero-filled hash object. */ Hash(HashType type); + /* Check whether a hash is set. */ + operator bool () const { return type != htUnknown; } + /* Check whether two hash are equal. */ bool operator == (const Hash & h2) const; @@ -39,18 +46,31 @@ struct Hash /* For sorting. */ bool operator < (const Hash & h) const; + + /* Returns the length of a base-16 representation of this hash. */ + size_t base16Len() const + { + return hashSize * 2; + } + + /* Returns the length of a base-32 representation of this hash. */ + size_t base32Len() const + { + return (hashSize * 8 - 1) / 5 + 1; + } + + std::string to_string(bool base32 = true) const; }; /* Convert a hash to a hexadecimal representation. */ string printHash(const Hash & hash); +Hash parseHash(const string & s); + /* Parse a hexadecimal representation of a hash code. */ Hash parseHash(HashType ht, const string & s); -/* Returns the length of a base-32 hash representation. */ -unsigned int hashLength32(const Hash & hash); - /* Convert a hash to a base-32 representation. */ string printHash32(const Hash & hash); diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 8af2e78d9ce4..98cad00d6d95 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,10 +6,6 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -ifeq ($(HAVE_OPENSSL), 1) - libutil_LDFLAGS = $(OPENSSL_LIBS) -else - libutil_SOURCES += $(d)/md5.c $(d)/sha1.c $(d)/sha256.c -endif +libutil_LDFLAGS = -llzma -lbz2 -pthread $(OPENSSL_LIBS) libutil_LIBS = libformat diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc new file mode 100644 index 000000000000..15bb1e175da6 --- /dev/null +++ b/src/libutil/logging.cc @@ -0,0 +1,79 @@ +#include "logging.hh" +#include "util.hh" + +namespace nix { + +Logger * logger = 0; + +class SimpleLogger : public Logger +{ +public: + + bool systemd, tty; + + SimpleLogger() + { + systemd = getEnv("IN_SYSTEMD") == "1"; + tty = isatty(STDERR_FILENO); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + + std::string prefix; + + if (systemd) { + char c; + switch (lvl) { + case lvlError: c = '3'; break; + case lvlInfo: c = '5'; break; + case lvlTalkative: case lvlChatty: c = '6'; break; + default: c = '7'; + } + prefix = std::string("<") + c + ">"; + } + + writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n"); + } + + void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override + { + log(lvl, fs); + } + + void stopActivity(Activity & activity) override + { + } +}; + +Verbosity verbosity = lvlInfo; + +void warnOnce(bool & haveWarned, const FormatOrString & fs) +{ + if (!haveWarned) { + printMsg(lvlError, format("warning: %1%") % fs.s); + haveWarned = true; + } +} + +void writeToStderr(const string & s) +{ + try { + writeFull(STDERR_FILENO, s); + } catch (SysError & e) { + /* Ignore failing writes to stderr if we're in an exception + handler, otherwise throw an exception. We need to ignore + write errors in exception handlers to ensure that cleanup + code runs to completion if the other side of stderr has + been closed unexpectedly. */ + if (!std::uncaught_exception()) throw; + } +} + +Logger * makeDefaultLogger() +{ + return new SimpleLogger(); +} + +} diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh new file mode 100644 index 000000000000..277dff280053 --- /dev/null +++ b/src/libutil/logging.hh @@ -0,0 +1,82 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +typedef enum { + lvlError = 0, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit +} Verbosity; + +class Activity; + +class Logger +{ + friend class Activity; + +public: + + virtual ~Logger() { } + + virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; + + void log(const FormatOrString & fs) + { + log(lvlInfo, fs); + } + + virtual void setExpected(const std::string & label, uint64_t value = 1) { } + virtual void setProgress(const std::string & label, uint64_t value = 1) { } + virtual void incExpected(const std::string & label, uint64_t value = 1) { } + virtual void incProgress(const std::string & label, uint64_t value = 1) { } + +private: + + virtual void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) = 0; + + virtual void stopActivity(Activity & activity) = 0; + +}; + +class Activity +{ +public: + Logger & logger; + + Activity(Logger & logger, Verbosity lvl, const FormatOrString & fs) + : logger(logger) + { + logger.startActivity(*this, lvl, fs); + } + + ~Activity() + { + logger.stopActivity(*this); + } +}; + +extern Logger * logger; + +Logger * makeDefaultLogger(); + +extern Verbosity verbosity; /* suppress msgs > this */ + +#define printMsg(level, f) \ + do { \ + if (level <= nix::verbosity) { \ + logger->log(level, (f)); \ + } \ + } while (0) + +#define debug(f) printMsg(lvlDebug, f) + +void warnOnce(bool & haveWarned, const FormatOrString & fs); + +void writeToStderr(const string & s); + +} diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh new file mode 100644 index 000000000000..35983aa2c918 --- /dev/null +++ b/src/libutil/lru-cache.hh @@ -0,0 +1,90 @@ +#pragma once + +#include <map> +#include <list> + +namespace nix { + +/* A simple least-recently used cache. Not thread-safe. */ +template<typename Key, typename Value> +class LRUCache +{ +private: + + size_t maxSize; + + // Stupid wrapper to get around circular dependency between Data + // and LRU. + struct LRUIterator; + + using Data = std::map<Key, std::pair<LRUIterator, Value>>; + using LRU = std::list<typename Data::iterator>; + + struct LRUIterator { typename LRU::iterator it; }; + + Data data; + LRU lru; + +public: + + LRUCache(size_t maxSize) : maxSize(maxSize) { } + + /* Insert or upsert an item in the cache. */ + void upsert(const Key & key, const Value & value) + { + erase(key); + + if (data.size() >= maxSize) { + /* Retire the oldest item. */ + auto oldest = lru.begin(); + data.erase(*oldest); + lru.erase(oldest); + } + + auto res = data.emplace(key, std::make_pair(LRUIterator(), value)); + assert(res.second); + auto & i(res.first); + + auto j = lru.insert(lru.end(), i); + + i->second.first.it = j; + } + + bool erase(const Key & key) + { + auto i = data.find(key); + if (i == data.end()) return false; + lru.erase(i->second.first.it); + data.erase(i); + return true; + } + + /* Look up an item in the cache. If it exists, it becomes the most + recently used item. */ + // FIXME: use boost::optional? + Value * get(const Key & key) + { + auto i = data.find(key); + if (i == data.end()) return 0; + + /* Move this item to the back of the LRU list. */ + lru.erase(i->second.first.it); + auto j = lru.insert(lru.end(), i); + i->second.first.it = j; + + return &i->second.second; + } + + size_t size() + { + return data.size(); + } + + void clear() + { + data.clear(); + lru.clear(); + } +}; + +} diff --git a/src/libutil/md32_common.h b/src/libutil/md32_common.h deleted file mode 100644 index 0cbcfaf8a20b..000000000000 --- a/src/libutil/md32_common.h +++ /dev/null @@ -1,620 +0,0 @@ -/* crypto/md32_common.h */ -/* ==================================================================== - * Copyright (c) 1999-2002 The OpenSSL Project. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. All advertising materials mentioning features or use of this - * software must display the following acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" - * - * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For written permission, please contact - * licensing@OpenSSL.org. - * - * 5. Products derived from this software may not be called "OpenSSL" - * nor may "OpenSSL" appear in their names without prior written - * permission of the OpenSSL Project. - * - * 6. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" - * - * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY - * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR - * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * ==================================================================== - * - * This product includes cryptographic software written by Eric Young - * (eay@cryptsoft.com). This product includes software written by Tim - * Hudson (tjh@cryptsoft.com). - * - */ - -/* - * This is a generic 32 bit "collector" for message digest algorithms. - * Whenever needed it collects input character stream into chunks of - * 32 bit values and invokes a block function that performs actual hash - * calculations. - * - * Porting guide. - * - * Obligatory macros: - * - * DATA_ORDER_IS_BIG_ENDIAN or DATA_ORDER_IS_LITTLE_ENDIAN - * this macro defines byte order of input stream. - * HASH_CBLOCK - * size of a unit chunk HASH_BLOCK operates on. - * HASH_LONG - * has to be at lest 32 bit wide, if it's wider, then - * HASH_LONG_LOG2 *has to* be defined along - * HASH_CTX - * context structure that at least contains following - * members: - * typedef struct { - * ... - * HASH_LONG Nl,Nh; - * HASH_LONG data[HASH_LBLOCK]; - * unsigned int num; - * ... - * } HASH_CTX; - * HASH_UPDATE - * name of "Update" function, implemented here. - * HASH_TRANSFORM - * name of "Transform" function, implemented here. - * HASH_FINAL - * name of "Final" function, implemented here. - * HASH_BLOCK_HOST_ORDER - * name of "block" function treating *aligned* input message - * in host byte order, implemented externally. - * HASH_BLOCK_DATA_ORDER - * name of "block" function treating *unaligned* input message - * in original (data) byte order, implemented externally (it - * actually is optional if data and host are of the same - * "endianess"). - * HASH_MAKE_STRING - * macro convering context variables to an ASCII hash string. - * - * Optional macros: - * - * B_ENDIAN or L_ENDIAN - * defines host byte-order. - * HASH_LONG_LOG2 - * defaults to 2 if not states otherwise. - * HASH_LBLOCK - * assumed to be HASH_CBLOCK/4 if not stated otherwise. - * HASH_BLOCK_DATA_ORDER_ALIGNED - * alternative "block" function capable of treating - * aligned input message in original (data) order, - * implemented externally. - * - * MD5 example: - * - * #define DATA_ORDER_IS_LITTLE_ENDIAN - * - * #define HASH_LONG MD5_LONG - * #define HASH_LONG_LOG2 MD5_LONG_LOG2 - * #define HASH_CTX MD5_CTX - * #define HASH_CBLOCK MD5_CBLOCK - * #define HASH_LBLOCK MD5_LBLOCK - * #define HASH_UPDATE MD5_Update - * #define HASH_TRANSFORM MD5_Transform - * #define HASH_FINAL MD5_Final - * #define HASH_BLOCK_HOST_ORDER md5_block_host_order - * #define HASH_BLOCK_DATA_ORDER md5_block_data_order - * - * <appro@fy.chalmers.se> - */ - -#if !defined(DATA_ORDER_IS_BIG_ENDIAN) && !defined(DATA_ORDER_IS_LITTLE_ENDIAN) -#error "DATA_ORDER must be defined!" -#endif - -#ifndef HASH_CBLOCK -#error "HASH_CBLOCK must be defined!" -#endif -#ifndef HASH_LONG -#error "HASH_LONG must be defined!" -#endif -#ifndef HASH_CTX -#error "HASH_CTX must be defined!" -#endif - -#ifndef HASH_UPDATE -#error "HASH_UPDATE must be defined!" -#endif -#ifndef HASH_TRANSFORM -#error "HASH_TRANSFORM must be defined!" -#endif -#ifndef HASH_FINAL -#error "HASH_FINAL must be defined!" -#endif - -#ifndef HASH_BLOCK_HOST_ORDER -#error "HASH_BLOCK_HOST_ORDER must be defined!" -#endif - -#if 0 -/* - * Moved below as it's required only if HASH_BLOCK_DATA_ORDER_ALIGNED - * isn't defined. - */ -#ifndef HASH_BLOCK_DATA_ORDER -#error "HASH_BLOCK_DATA_ORDER must be defined!" -#endif -#endif - -#ifndef HASH_LBLOCK -#define HASH_LBLOCK (HASH_CBLOCK/4) -#endif - -#ifndef HASH_LONG_LOG2 -#define HASH_LONG_LOG2 2 -#endif - -/* - * Engage compiler specific rotate intrinsic function if available. - */ -#undef ROTATE -#ifndef PEDANTIC -# if defined(_MSC_VER) || defined(__ICC) -# define ROTATE(a,n) _lrotl(a,n) -# elif defined(__MWERKS__) -# if defined(__POWERPC__) -# define ROTATE(a,n) __rlwinm(a,n,0,31) -# elif defined(__MC68K__) - /* Motorola specific tweak. <appro@fy.chalmers.se> */ -# define ROTATE(a,n) ( n<24 ? __rol(a,n) : __ror(a,32-n) ) -# else -# define ROTATE(a,n) __rol(a,n) -# endif -# elif defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) - /* - * Some GNU C inline assembler templates. Note that these are - * rotates by *constant* number of bits! But that's exactly - * what we need here... - * <appro@fy.chalmers.se> - */ -# if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) -# define ROTATE(a,n) ({ register unsigned int ret; \ - asm ( \ - "roll %1,%0" \ - : "=r"(ret) \ - : "I"(n), "0"(a) \ - : "cc"); \ - ret; \ - }) -# elif defined(__powerpc) || defined(__ppc__) || defined(__powerpc64__) -# define ROTATE(a,n) ({ register unsigned int ret; \ - asm ( \ - "rlwinm %0,%1,%2,0,31" \ - : "=r"(ret) \ - : "r"(a), "I"(n)); \ - ret; \ - }) -# endif -# endif -#endif /* PEDANTIC */ - -#if HASH_LONG_LOG2==2 /* Engage only if sizeof(HASH_LONG)== 4 */ -/* A nice byte order reversal from Wei Dai <weidai@eskimo.com> */ -#ifdef ROTATE -/* 5 instructions with rotate instruction, else 9 */ -#define REVERSE_FETCH32(a,l) ( \ - l=*(const HASH_LONG *)(a), \ - ((ROTATE(l,8)&0x00FF00FF)|(ROTATE((l&0x00FF00FF),24))) \ - ) -#else -/* 6 instructions with rotate instruction, else 8 */ -#define REVERSE_FETCH32(a,l) ( \ - l=*(const HASH_LONG *)(a), \ - l=(((l>>8)&0x00FF00FF)|((l&0x00FF00FF)<<8)), \ - ROTATE(l,16) \ - ) -/* - * Originally the middle line started with l=(((l&0xFF00FF00)>>8)|... - * It's rewritten as above for two reasons: - * - RISCs aren't good at long constants and have to explicitely - * compose 'em with several (well, usually 2) instructions in a - * register before performing the actual operation and (as you - * already realized:-) having same constant should inspire the - * compiler to permanently allocate the only register for it; - * - most modern CPUs have two ALUs, but usually only one has - * circuitry for shifts:-( this minor tweak inspires compiler - * to schedule shift instructions in a better way... - * - * <appro@fy.chalmers.se> - */ -#endif -#endif - -#ifndef ROTATE -#define ROTATE(a,n) (((a)<<(n))|(((a)&0xffffffff)>>(32-(n)))) -#endif - -/* - * Make some obvious choices. E.g., HASH_BLOCK_DATA_ORDER_ALIGNED - * and HASH_BLOCK_HOST_ORDER ought to be the same if input data - * and host are of the same "endianess". It's possible to mask - * this with blank #define HASH_BLOCK_DATA_ORDER though... - * - * <appro@fy.chalmers.se> - */ -#if defined(B_ENDIAN) -# if defined(DATA_ORDER_IS_BIG_ENDIAN) -# if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) && HASH_LONG_LOG2==2 -# define HASH_BLOCK_DATA_ORDER_ALIGNED HASH_BLOCK_HOST_ORDER -# endif -# endif -#elif defined(L_ENDIAN) -# if defined(DATA_ORDER_IS_LITTLE_ENDIAN) -# if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) && HASH_LONG_LOG2==2 -# define HASH_BLOCK_DATA_ORDER_ALIGNED HASH_BLOCK_HOST_ORDER -# endif -# endif -#endif - -#if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) -#ifndef HASH_BLOCK_DATA_ORDER -#error "HASH_BLOCK_DATA_ORDER must be defined!" -#endif -#endif - -#if defined(DATA_ORDER_IS_BIG_ENDIAN) - -#ifndef PEDANTIC -# if defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) -# if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) - /* - * This gives ~30-40% performance improvement in SHA-256 compiled - * with gcc [on P4]. Well, first macro to be frank. We can pull - * this trick on x86* platforms only, because these CPUs can fetch - * unaligned data without raising an exception. - */ -# define HOST_c2l(c,l) ({ unsigned int r=*((const unsigned int *)(c)); \ - asm ("bswapl %0":"=r"(r):"0"(r)); \ - (c)+=4; (l)=r; }) -# define HOST_l2c(l,c) ({ unsigned int r=(l); \ - asm ("bswapl %0":"=r"(r):"0"(r)); \ - *((unsigned int *)(c))=r; (c)+=4; r; }) -# endif -# endif -#endif - -#ifndef HOST_c2l -#define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++)))<<24), \ - l|=(((unsigned long)(*((c)++)))<<16), \ - l|=(((unsigned long)(*((c)++)))<< 8), \ - l|=(((unsigned long)(*((c)++))) ), \ - l) -#endif -#define HOST_p_c2l(c,l,n) { \ - switch (n) { \ - case 0: l =((unsigned long)(*((c)++)))<<24; \ - case 1: l|=((unsigned long)(*((c)++)))<<16; \ - case 2: l|=((unsigned long)(*((c)++)))<< 8; \ - case 3: l|=((unsigned long)(*((c)++))); \ - } } -#define HOST_p_c2l_p(c,l,sc,len) { \ - switch (sc) { \ - case 0: l =((unsigned long)(*((c)++)))<<24; \ - if (--len == 0) break; \ - case 1: l|=((unsigned long)(*((c)++)))<<16; \ - if (--len == 0) break; \ - case 2: l|=((unsigned long)(*((c)++)))<< 8; \ - } } -/* NOTE the pointer is not incremented at the end of this */ -#define HOST_c2l_p(c,l,n) { \ - l=0; (c)+=n; \ - switch (n) { \ - case 3: l =((unsigned long)(*(--(c))))<< 8; \ - case 2: l|=((unsigned long)(*(--(c))))<<16; \ - case 1: l|=((unsigned long)(*(--(c))))<<24; \ - } } -#ifndef HOST_l2c -#define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l)>>24)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16)&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ - *((c)++)=(unsigned char)(((l) )&0xff), \ - l) -#endif - -#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) - -#if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) - /* See comment in DATA_ORDER_IS_BIG_ENDIAN section. */ -# define HOST_c2l(c,l) ((l)=*((const unsigned int *)(c)), (c)+=4, l) -# define HOST_l2c(l,c) (*((unsigned int *)(c))=(l), (c)+=4, l) -#endif - -#ifndef HOST_c2l -#define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++))) ), \ - l|=(((unsigned long)(*((c)++)))<< 8), \ - l|=(((unsigned long)(*((c)++)))<<16), \ - l|=(((unsigned long)(*((c)++)))<<24), \ - l) -#endif -#define HOST_p_c2l(c,l,n) { \ - switch (n) { \ - case 0: l =((unsigned long)(*((c)++))); \ - case 1: l|=((unsigned long)(*((c)++)))<< 8; \ - case 2: l|=((unsigned long)(*((c)++)))<<16; \ - case 3: l|=((unsigned long)(*((c)++)))<<24; \ - } } -#define HOST_p_c2l_p(c,l,sc,len) { \ - switch (sc) { \ - case 0: l =((unsigned long)(*((c)++))); \ - if (--len == 0) break; \ - case 1: l|=((unsigned long)(*((c)++)))<< 8; \ - if (--len == 0) break; \ - case 2: l|=((unsigned long)(*((c)++)))<<16; \ - } } -/* NOTE the pointer is not incremented at the end of this */ -#define HOST_c2l_p(c,l,n) { \ - l=0; (c)+=n; \ - switch (n) { \ - case 3: l =((unsigned long)(*(--(c))))<<16; \ - case 2: l|=((unsigned long)(*(--(c))))<< 8; \ - case 1: l|=((unsigned long)(*(--(c)))); \ - } } -#ifndef HOST_l2c -#define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16)&0xff), \ - *((c)++)=(unsigned char)(((l)>>24)&0xff), \ - l) -#endif - -#endif - -/* - * Time for some action:-) - */ - -int HASH_UPDATE (HASH_CTX *c, const void *data_, size_t len) - { - const unsigned char *data=data_; - register HASH_LONG * p; - register HASH_LONG l; - size_t sw,sc,ew,ec; - - if (len==0) return 1; - - l=(c->Nl+(((HASH_LONG)len)<<3))&0xffffffffUL; - /* 95-05-24 eay Fixed a bug with the overflow handling, thanks to - * Wei Dai <weidai@eskimo.com> for pointing it out. */ - if (l < c->Nl) /* overflow */ - c->Nh++; - c->Nh+=(len>>29); /* might cause compiler warning on 16-bit */ - c->Nl=l; - - if (c->num != 0) - { - p=c->data; - sw=c->num>>2; - sc=c->num&0x03; - - if ((c->num+len) >= HASH_CBLOCK) - { - l=p[sw]; HOST_p_c2l(data,l,sc); p[sw++]=l; - for (; sw<HASH_LBLOCK; sw++) - { - HOST_c2l(data,l); p[sw]=l; - } - HASH_BLOCK_HOST_ORDER (c,p,1); - len-=(HASH_CBLOCK-c->num); - c->num=0; - /* drop through and do the rest */ - } - else - { - c->num+=(unsigned int)len; - if ((sc+len) < 4) /* ugly, add char's to a word */ - { - l=p[sw]; HOST_p_c2l_p(data,l,sc,len); p[sw]=l; - } - else - { - ew=(c->num>>2); - ec=(c->num&0x03); - if (sc) - l=p[sw]; - HOST_p_c2l(data,l,sc); - p[sw++]=l; - for (; sw < ew; sw++) - { - HOST_c2l(data,l); p[sw]=l; - } - if (ec) - { - HOST_c2l_p(data,l,ec); p[sw]=l; - } - } - return 1; - } - } - - sw=len/HASH_CBLOCK; - if (sw > 0) - { -#if defined(HASH_BLOCK_DATA_ORDER_ALIGNED) - /* - * Note that HASH_BLOCK_DATA_ORDER_ALIGNED gets defined - * only if sizeof(HASH_LONG)==4. - */ - if ((((size_t)data)%4) == 0) - { - /* data is properly aligned so that we can cast it: */ - HASH_BLOCK_DATA_ORDER_ALIGNED (c,(const HASH_LONG *)data,sw); - sw*=HASH_CBLOCK; - data+=sw; - len-=sw; - } - else -#if !defined(HASH_BLOCK_DATA_ORDER) - while (sw--) - { - memcpy (p=c->data,data,HASH_CBLOCK); - HASH_BLOCK_DATA_ORDER_ALIGNED(c,p,1); - data+=HASH_CBLOCK; - len-=HASH_CBLOCK; - } -#endif -#endif -#if defined(HASH_BLOCK_DATA_ORDER) - { - HASH_BLOCK_DATA_ORDER(c,data,sw); - sw*=HASH_CBLOCK; - data+=sw; - len-=sw; - } -#endif - } - - if (len!=0) - { - p = c->data; - c->num = len; - ew=len>>2; /* words to copy */ - ec=len&0x03; - for (; ew; ew--,p++) - { - HOST_c2l(data,l); *p=l; - } - HOST_c2l_p(data,l,ec); - *p=l; - } - return 1; - } - - -void HASH_TRANSFORM (HASH_CTX *c, const unsigned char *data) - { -#if defined(HASH_BLOCK_DATA_ORDER_ALIGNED) - if ((((size_t)data)%4) == 0) - /* data is properly aligned so that we can cast it: */ - HASH_BLOCK_DATA_ORDER_ALIGNED (c,(const HASH_LONG *)data,1); - else -#if !defined(HASH_BLOCK_DATA_ORDER) - { - memcpy (c->data,data,HASH_CBLOCK); - HASH_BLOCK_DATA_ORDER_ALIGNED (c,c->data,1); - } -#endif -#endif -#if defined(HASH_BLOCK_DATA_ORDER) - HASH_BLOCK_DATA_ORDER (c,data,1); -#endif - } - - -int HASH_FINAL (unsigned char *md, HASH_CTX *c) - { - register HASH_LONG *p; - register unsigned long l; - register int i,j; - static const unsigned char end[4]={0x80,0x00,0x00,0x00}; - const unsigned char *cp=end; - - /* c->num should definitly have room for at least one more byte. */ - p=c->data; - i=c->num>>2; - j=c->num&0x03; - -#if 0 - /* purify often complains about the following line as an - * Uninitialized Memory Read. While this can be true, the - * following p_c2l macro will reset l when that case is true. - * This is because j&0x03 contains the number of 'valid' bytes - * already in p[i]. If and only if j&0x03 == 0, the UMR will - * occur but this is also the only time p_c2l will do - * l= *(cp++) instead of l|= *(cp++) - * Many thanks to Alex Tang <altitude@cic.net> for pickup this - * 'potential bug' */ -#ifdef PURIFY - if (j==0) p[i]=0; /* Yeah, but that's not the way to fix it:-) */ -#endif - l=p[i]; -#else - l = (j==0) ? 0 : p[i]; -#endif - HOST_p_c2l(cp,l,j); p[i++]=l; /* i is the next 'undefined word' */ - - if (i>(HASH_LBLOCK-2)) /* save room for Nl and Nh */ - { - if (i<HASH_LBLOCK) p[i]=0; - HASH_BLOCK_HOST_ORDER (c,p,1); - i=0; - } - for (; i<(HASH_LBLOCK-2); i++) - p[i]=0; - -#if defined(DATA_ORDER_IS_BIG_ENDIAN) - p[HASH_LBLOCK-2]=c->Nh; - p[HASH_LBLOCK-1]=c->Nl; -#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) - p[HASH_LBLOCK-2]=c->Nl; - p[HASH_LBLOCK-1]=c->Nh; -#endif - HASH_BLOCK_HOST_ORDER (c,p,1); - -#ifndef HASH_MAKE_STRING -#error "HASH_MAKE_STRING must be defined!" -#else - HASH_MAKE_STRING(c,md); -#endif - - c->num=0; - /* clear stuff, HASH_BLOCK may be leaving some stuff on the stack - * but I'm not worried :-) - OPENSSL_cleanse((void *)c,sizeof(HASH_CTX)); - */ - return 1; - } - -#ifndef MD32_REG_T -#define MD32_REG_T long -/* - * This comment was originaly written for MD5, which is why it - * discusses A-D. But it basically applies to all 32-bit digests, - * which is why it was moved to common header file. - * - * In case you wonder why A-D are declared as long and not - * as MD5_LONG. Doing so results in slight performance - * boost on LP64 architectures. The catch is we don't - * really care if 32 MSBs of a 64-bit register get polluted - * with eventual overflows as we *save* only 32 LSBs in - * *either* case. Now declaring 'em long excuses the compiler - * from keeping 32 MSBs zeroed resulting in 13% performance - * improvement under SPARC Solaris7/64 and 5% under AlphaLinux. - * Well, to be honest it should say that this *prevents* - * performance degradation. - * <appro@fy.chalmers.se> - * Apparently there're LP64 compilers that generate better - * code if A-D are declared int. Most notably GCC-x86_64 - * generates better code. - * <appro@fy.chalmers.se> - */ -#endif diff --git a/src/libutil/md5.c b/src/libutil/md5.c deleted file mode 100644 index b31640cdcced..000000000000 --- a/src/libutil/md5.c +++ /dev/null @@ -1,365 +0,0 @@ -/* Functions to compute MD5 message digest of files or memory blocks. - according to the definition of MD5 in RFC 1321 from April 1992. - Copyright (C) 1995,1996,1997,1999,2000,2001 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, write to the Free - Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA - 02111-1307 USA. */ - -/* Written by Ulrich Drepper <drepper@gnu.ai.mit.edu>, 1995. */ - -#include <sys/types.h> - -#include <stdlib.h> -#include <string.h> - -#include "md5.h" - - -static md5_uint32 SWAP(md5_uint32 n) -{ - static int checked = 0; - static int bigendian = 0; - static md5_uint32 test; - - if (!checked) { - test = 1; - if (* (char *) &test == 0) - bigendian = 1; - checked = 1; - } - - if (bigendian) - return (((n) << 24) | (((n) & 0xff00) << 8) | (((n) >> 8) & 0xff00) | ((n) >> 24)); - else - return n; -} - - -/* This array contains the bytes used to pad the buffer to the next - 64-byte boundary. (RFC 1321, 3.1: Step 1) */ -static const unsigned char fillbuf[64] = { 0x80, 0 /* , 0, 0, ... */ }; - - -/* Initialize structure containing state of computation. - (RFC 1321, 3.3: Step 3) */ -void -MD5_Init (ctx) - struct MD5_CTX *ctx; -{ - ctx->A = 0x67452301; - ctx->B = 0xefcdab89; - ctx->C = 0x98badcfe; - ctx->D = 0x10325476; - - ctx->total[0] = ctx->total[1] = 0; - ctx->buflen = 0; -} - -/* Put result from CTX in first 16 bytes following RESBUF. The result - must be in little endian byte order. - - IMPORTANT: On some systems it is required that RESBUF is correctly - aligned for a 32 bits value. */ -void * -md5_read_ctx (ctx, resbuf) - const struct MD5_CTX *ctx; - void *resbuf; -{ - ((md5_uint32 *) resbuf)[0] = SWAP (ctx->A); - ((md5_uint32 *) resbuf)[1] = SWAP (ctx->B); - ((md5_uint32 *) resbuf)[2] = SWAP (ctx->C); - ((md5_uint32 *) resbuf)[3] = SWAP (ctx->D); - - return resbuf; -} - -/* Process the remaining bytes in the internal buffer and the usual - prolog according to the standard and write the result to RESBUF. - - IMPORTANT: On some systems it is required that RESBUF is correctly - aligned for a 32 bits value. */ -void * -MD5_Final (resbuf, ctx) - void *resbuf; - struct MD5_CTX *ctx; -{ - /* Take yet unprocessed bytes into account. */ - md5_uint32 bytes = ctx->buflen; - size_t pad; - - /* Now count remaining bytes. */ - ctx->total[0] += bytes; - if (ctx->total[0] < bytes) - ++ctx->total[1]; - - pad = bytes >= 56 ? 64 + 56 - bytes : 56 - bytes; - memcpy (&ctx->buffer[bytes], fillbuf, pad); - - /* Put the 64-bit file length in *bits* at the end of the buffer. */ - *(md5_uint32 *) &ctx->buffer[bytes + pad] = SWAP (ctx->total[0] << 3); - *(md5_uint32 *) &ctx->buffer[bytes + pad + 4] = SWAP ((ctx->total[1] << 3) | - (ctx->total[0] >> 29)); - - /* Process last bytes. */ - md5_process_block (ctx->buffer, bytes + pad + 8, ctx); - - return md5_read_ctx (ctx, resbuf); -} - -void -MD5_Update (ctx, buffer, len) - struct MD5_CTX *ctx; - const void *buffer; - size_t len; -{ - /* When we already have some bits in our internal buffer concatenate - both inputs first. */ - if (ctx->buflen != 0) - { - size_t left_over = ctx->buflen; - size_t add = 128 - left_over > len ? len : 128 - left_over; - - memcpy (&ctx->buffer[left_over], buffer, add); - ctx->buflen += add; - - if (ctx->buflen > 64) - { - md5_process_block (ctx->buffer, ctx->buflen & ~63, ctx); - - ctx->buflen &= 63; - /* The regions in the following copy operation cannot overlap. */ - memcpy (ctx->buffer, &ctx->buffer[(left_over + add) & ~63], - ctx->buflen); - } - - buffer = (const char *) buffer + add; - len -= add; - } - - /* Process available complete blocks. */ - if (len >= 64) - { -#if !_STRING_ARCH_unaligned -/* To check alignment gcc has an appropriate operator. Other - compilers don't. */ -# if __GNUC__ >= 2 -# define UNALIGNED_P(p) (((md5_uintptr) p) % __alignof__ (md5_uint32) != 0) -# else -# define UNALIGNED_P(p) (((md5_uintptr) p) % sizeof (md5_uint32) != 0) -# endif - if (UNALIGNED_P (buffer)) - while (len > 64) - { - md5_process_block (memcpy (ctx->buffer, buffer, 64), 64, ctx); - buffer = (const char *) buffer + 64; - len -= 64; - } - else -#endif - { - md5_process_block (buffer, len & ~63, ctx); - buffer = (const char *) buffer + (len & ~63); - len &= 63; - } - } - - /* Move remaining bytes in internal buffer. */ - if (len > 0) - { - size_t left_over = ctx->buflen; - - memcpy (&ctx->buffer[left_over], buffer, len); - left_over += len; - if (left_over >= 64) - { - md5_process_block (ctx->buffer, 64, ctx); - left_over -= 64; - memcpy (ctx->buffer, &ctx->buffer[64], left_over); - } - ctx->buflen = left_over; - } -} - - -/* These are the four functions used in the four steps of the MD5 algorithm - and defined in the RFC 1321. The first function is a little bit optimized - (as found in Colin Plumbs public domain implementation). */ -/* #define FF(b, c, d) ((b & c) | (~b & d)) */ -#define FF(b, c, d) (d ^ (b & (c ^ d))) -#define FG(b, c, d) FF (d, b, c) -#define FH(b, c, d) (b ^ c ^ d) -#define FI(b, c, d) (c ^ (b | ~d)) - -/* Process LEN bytes of BUFFER, accumulating context into CTX. - It is assumed that LEN % 64 == 0. */ - -void -md5_process_block (buffer, len, ctx) - const void *buffer; - size_t len; - struct MD5_CTX *ctx; -{ - md5_uint32 correct_words[16]; - const md5_uint32 *words = buffer; - size_t nwords = len / sizeof (md5_uint32); - const md5_uint32 *endp = words + nwords; - md5_uint32 A = ctx->A; - md5_uint32 B = ctx->B; - md5_uint32 C = ctx->C; - md5_uint32 D = ctx->D; - - /* First increment the byte count. RFC 1321 specifies the possible - length of the file up to 2^64 bits. Here we only compute the - number of bytes. Do a double word increment. */ - ctx->total[0] += len; - if (ctx->total[0] < len) - ++ctx->total[1]; - - /* Process all bytes in the buffer with 64 bytes in each round of - the loop. */ - while (words < endp) - { - md5_uint32 *cwp = correct_words; - md5_uint32 A_save = A; - md5_uint32 B_save = B; - md5_uint32 C_save = C; - md5_uint32 D_save = D; - - /* First round: using the given function, the context and a constant - the next context is computed. Because the algorithms processing - unit is a 32-bit word and it is determined to work on words in - little endian byte order we perhaps have to change the byte order - before the computation. To reduce the work for the next steps - we store the swapped words in the array CORRECT_WORDS. */ - -#define OP(a, b, c, d, s, T) \ - do \ - { \ - a += FF (b, c, d) + (*cwp++ = SWAP (*words)) + T; \ - ++words; \ - CYCLIC (a, s); \ - a += b; \ - } \ - while (0) - - /* It is unfortunate that C does not provide an operator for - cyclic rotation. Hope the C compiler is smart enough. */ -#define CYCLIC(w, s) (w = (w << s) | (w >> (32 - s))) - - /* Before we start, one word to the strange constants. - They are defined in RFC 1321 as - - T[i] = (int) (4294967296.0 * fabs (sin (i))), i=1..64 - */ - - /* Round 1. */ - OP (A, B, C, D, 7, 0xd76aa478); - OP (D, A, B, C, 12, 0xe8c7b756); - OP (C, D, A, B, 17, 0x242070db); - OP (B, C, D, A, 22, 0xc1bdceee); - OP (A, B, C, D, 7, 0xf57c0faf); - OP (D, A, B, C, 12, 0x4787c62a); - OP (C, D, A, B, 17, 0xa8304613); - OP (B, C, D, A, 22, 0xfd469501); - OP (A, B, C, D, 7, 0x698098d8); - OP (D, A, B, C, 12, 0x8b44f7af); - OP (C, D, A, B, 17, 0xffff5bb1); - OP (B, C, D, A, 22, 0x895cd7be); - OP (A, B, C, D, 7, 0x6b901122); - OP (D, A, B, C, 12, 0xfd987193); - OP (C, D, A, B, 17, 0xa679438e); - OP (B, C, D, A, 22, 0x49b40821); - - /* For the second to fourth round we have the possibly swapped words - in CORRECT_WORDS. Redefine the macro to take an additional first - argument specifying the function to use. */ -#undef OP -#define OP(f, a, b, c, d, k, s, T) \ - do \ - { \ - a += f (b, c, d) + correct_words[k] + T; \ - CYCLIC (a, s); \ - a += b; \ - } \ - while (0) - - /* Round 2. */ - OP (FG, A, B, C, D, 1, 5, 0xf61e2562); - OP (FG, D, A, B, C, 6, 9, 0xc040b340); - OP (FG, C, D, A, B, 11, 14, 0x265e5a51); - OP (FG, B, C, D, A, 0, 20, 0xe9b6c7aa); - OP (FG, A, B, C, D, 5, 5, 0xd62f105d); - OP (FG, D, A, B, C, 10, 9, 0x02441453); - OP (FG, C, D, A, B, 15, 14, 0xd8a1e681); - OP (FG, B, C, D, A, 4, 20, 0xe7d3fbc8); - OP (FG, A, B, C, D, 9, 5, 0x21e1cde6); - OP (FG, D, A, B, C, 14, 9, 0xc33707d6); - OP (FG, C, D, A, B, 3, 14, 0xf4d50d87); - OP (FG, B, C, D, A, 8, 20, 0x455a14ed); - OP (FG, A, B, C, D, 13, 5, 0xa9e3e905); - OP (FG, D, A, B, C, 2, 9, 0xfcefa3f8); - OP (FG, C, D, A, B, 7, 14, 0x676f02d9); - OP (FG, B, C, D, A, 12, 20, 0x8d2a4c8a); - - /* Round 3. */ - OP (FH, A, B, C, D, 5, 4, 0xfffa3942); - OP (FH, D, A, B, C, 8, 11, 0x8771f681); - OP (FH, C, D, A, B, 11, 16, 0x6d9d6122); - OP (FH, B, C, D, A, 14, 23, 0xfde5380c); - OP (FH, A, B, C, D, 1, 4, 0xa4beea44); - OP (FH, D, A, B, C, 4, 11, 0x4bdecfa9); - OP (FH, C, D, A, B, 7, 16, 0xf6bb4b60); - OP (FH, B, C, D, A, 10, 23, 0xbebfbc70); - OP (FH, A, B, C, D, 13, 4, 0x289b7ec6); - OP (FH, D, A, B, C, 0, 11, 0xeaa127fa); - OP (FH, C, D, A, B, 3, 16, 0xd4ef3085); - OP (FH, B, C, D, A, 6, 23, 0x04881d05); - OP (FH, A, B, C, D, 9, 4, 0xd9d4d039); - OP (FH, D, A, B, C, 12, 11, 0xe6db99e5); - OP (FH, C, D, A, B, 15, 16, 0x1fa27cf8); - OP (FH, B, C, D, A, 2, 23, 0xc4ac5665); - - /* Round 4. */ - OP (FI, A, B, C, D, 0, 6, 0xf4292244); - OP (FI, D, A, B, C, 7, 10, 0x432aff97); - OP (FI, C, D, A, B, 14, 15, 0xab9423a7); - OP (FI, B, C, D, A, 5, 21, 0xfc93a039); - OP (FI, A, B, C, D, 12, 6, 0x655b59c3); - OP (FI, D, A, B, C, 3, 10, 0x8f0ccc92); - OP (FI, C, D, A, B, 10, 15, 0xffeff47d); - OP (FI, B, C, D, A, 1, 21, 0x85845dd1); - OP (FI, A, B, C, D, 8, 6, 0x6fa87e4f); - OP (FI, D, A, B, C, 15, 10, 0xfe2ce6e0); - OP (FI, C, D, A, B, 6, 15, 0xa3014314); - OP (FI, B, C, D, A, 13, 21, 0x4e0811a1); - OP (FI, A, B, C, D, 4, 6, 0xf7537e82); - OP (FI, D, A, B, C, 11, 10, 0xbd3af235); - OP (FI, C, D, A, B, 2, 15, 0x2ad7d2bb); - OP (FI, B, C, D, A, 9, 21, 0xeb86d391); - - /* Add the starting values of the context. */ - A += A_save; - B += B_save; - C += C_save; - D += D_save; - } - - /* Put checksum in context given as argument. */ - ctx->A = A; - ctx->B = B; - ctx->C = C; - ctx->D = D; -} diff --git a/src/libutil/md5.h b/src/libutil/md5.h deleted file mode 100644 index 228d4972320f..000000000000 --- a/src/libutil/md5.h +++ /dev/null @@ -1,82 +0,0 @@ -/* Declaration of functions and data types used for MD5 sum computing - library functions. - Copyright (C) 1995,1996,1997,1999,2000,2001 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, write to the Free - Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA - 02111-1307 USA. */ - -#ifndef _MD5_H -#define _MD5_H 1 - -#include <inttypes.h> -typedef uint32_t md5_uint32; -typedef uintptr_t md5_uintptr; - -/* Structure to save state of computation between the single steps. */ -struct MD5_CTX -{ - md5_uint32 A; - md5_uint32 B; - md5_uint32 C; - md5_uint32 D; - - md5_uint32 total[2]; - md5_uint32 buflen; - char buffer[128] __attribute__ ((__aligned__ (__alignof__ (md5_uint32)))); -}; - -/* - * The following three functions are build up the low level used in - * the functions `md5_stream' and `md5_buffer'. - */ - -/* Initialize structure containing state of computation. - (RFC 1321, 3.3: Step 3) */ -extern void MD5_Init (struct MD5_CTX *ctx); - -/* Starting with the result of former calls of this function (or the - initialization function update the context for the next LEN bytes - starting at BUFFER. - It is necessary that LEN is a multiple of 64!!! */ -extern void md5_process_block (const void *buffer, size_t len, - struct MD5_CTX *ctx); - -/* Starting with the result of former calls of this function (or the - initialization function update the context for the next LEN bytes - starting at BUFFER. - It is NOT required that LEN is a multiple of 64. */ -extern void MD5_Update (struct MD5_CTX *ctx, const void *buffer, size_t len); - -/* Process the remaining bytes in the buffer and put result from CTX - in first 16 bytes following RESBUF. The result is always in little - endian byte order, so that a byte-wise output yields to the wanted - ASCII representation of the message digest. - - IMPORTANT: On some systems it is required that RESBUF is correctly - aligned for a 32 bits value. */ -extern void *MD5_Final (void *resbuf, struct MD5_CTX *ctx); - - -/* Put result from CTX in first 16 bytes following RESBUF. The result is - always in little endian byte order, so that a byte-wise output yields - to the wanted ASCII representation of the message digest. - - IMPORTANT: On some systems it is required that RESBUF is correctly - aligned for a 32 bits value. */ -extern void *md5_read_ctx (const struct MD5_CTX *ctx, void *resbuf); - - -#endif /* md5.h */ diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh new file mode 100644 index 000000000000..f291cd578388 --- /dev/null +++ b/src/libutil/pool.hh @@ -0,0 +1,151 @@ +#pragma once + +#include <functional> +#include <limits> +#include <list> +#include <memory> +#include <cassert> + +#include "sync.hh" +#include "ref.hh" + +namespace nix { + +/* This template class implements a simple pool manager of resources + of some type R, such as database connections. It is used as + follows: + + class Connection { ... }; + + Pool<Connection> pool; + + { + auto conn(pool.get()); + conn->exec("select ..."); + } + + Here, the Connection object referenced by ‘conn’ is automatically + returned to the pool when ‘conn’ goes out of scope. +*/ + +template <class R> +class Pool +{ +public: + + /* A function that produces new instances of R on demand. */ + typedef std::function<ref<R>()> Factory; + + /* A function that checks whether an instance of R is still + usable. Unusable instances are removed from the pool. */ + typedef std::function<bool(const ref<R> &)> Validator; + +private: + + Factory factory; + Validator validator; + + struct State + { + size_t inUse = 0; + size_t max; + std::vector<ref<R>> idle; + }; + + Sync<State> state; + + std::condition_variable wakeup; + +public: + + Pool(size_t max = std::numeric_limits<size_t>::max(), + const Factory & factory = []() { return make_ref<R>(); }, + const Validator & validator = [](ref<R> r) { return true; }) + : factory(factory) + , validator(validator) + { + auto state_(state.lock()); + state_->max = max; + } + + ~Pool() + { + auto state_(state.lock()); + assert(!state_->inUse); + state_->max = 0; + state_->idle.clear(); + } + + class Handle + { + private: + Pool & pool; + std::shared_ptr<R> r; + + friend Pool; + + Handle(Pool & pool, std::shared_ptr<R> r) : pool(pool), r(r) { } + + public: + Handle(Handle && h) : pool(h.pool), r(h.r) { h.r.reset(); } + + Handle(const Handle & l) = delete; + + ~Handle() + { + if (!r) return; + { + auto state_(pool.state.lock()); + state_->idle.push_back(ref<R>(r)); + assert(state_->inUse); + state_->inUse--; + } + pool.wakeup.notify_one(); + } + + R * operator -> () { return &*r; } + R & operator * () { return *r; } + }; + + Handle get() + { + { + auto state_(state.lock()); + + /* If we're over the maximum number of instance, we need + to wait until a slot becomes available. */ + while (state_->idle.empty() && state_->inUse >= state_->max) + state_.wait(wakeup); + + while (!state_->idle.empty()) { + auto p = state_->idle.back(); + state_->idle.pop_back(); + if (validator(p)) { + state_->inUse++; + return Handle(*this, p); + } + } + + state_->inUse++; + } + + /* We need to create a new instance. Because that might take a + while, we don't hold the lock in the meantime. */ + try { + Handle h(*this, factory()); + return h; + } catch (...) { + auto state_(state.lock()); + state_->inUse--; + throw; + } + } + + unsigned int count() + { + auto state_(state.lock()); + return state_->idle.size() + state_->inUse; + } +}; + +} diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh new file mode 100644 index 000000000000..85afa28119a9 --- /dev/null +++ b/src/libutil/ref.hh @@ -0,0 +1,81 @@ +#pragma once + +#include <memory> +#include <exception> +#include <stdexcept> + +namespace nix { + +/* A simple non-nullable reference-counted pointer. Actually a wrapper + around std::shared_ptr that prevents non-null constructions. */ +template<typename T> +class ref +{ +private: + + std::shared_ptr<T> p; + +public: + + ref<T>(const ref<T> & r) + : p(r.p) + { } + + explicit ref<T>(const std::shared_ptr<T> & p) + : p(p) + { + if (!p) + throw std::invalid_argument("null pointer cast to ref"); + } + + explicit ref<T>(T * p) + : p(p) + { + if (!p) + throw std::invalid_argument("null pointer cast to ref"); + } + + T* operator ->() const + { + return &*p; + } + + T& operator *() const + { + return *p; + } + + operator std::shared_ptr<T> () + { + return p; + } + + template<typename T2> + ref<T2> cast() + { + return ref<T2>(std::dynamic_pointer_cast<T2>(p)); + } + + template<typename T2> + operator ref<T2> () + { + return ref<T2>((std::shared_ptr<T2>) p); + } + +private: + + template<typename T2, typename... Args> + friend ref<T2> + make_ref(Args&&... args); + +}; + +template<typename T, typename... Args> +inline ref<T> +make_ref(Args&&... args) +{ + auto p = std::make_shared<T>(std::forward<Args>(args)...); + return ref<T>(p); +} + +} diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 92417507508a..5c45c890f7b6 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -16,11 +16,11 @@ BufferedSink::~BufferedSink() delete[] buffer; } - + void BufferedSink::operator () (const unsigned char * data, size_t len) { if (!buffer) buffer = new unsigned char[bufSize]; - + while (len) { /* Optimisation: bypass the buffer if the data exceeds the buffer size. */ @@ -64,15 +64,25 @@ static void warnLargeDump() void FdSink::write(const unsigned char * data, size_t len) { + written += len; static bool warned = false; if (warn && !warned) { - written += len; if (written > threshold) { warnLargeDump(); warned = true; } } - writeFull(fd, data, len); + try { + writeFull(fd, data, len); + } catch (SysError & e) { + _good = true; + } +} + + +bool FdSink::good() +{ + return _good; } @@ -96,7 +106,7 @@ size_t BufferedSource::read(unsigned char * data, size_t len) if (!buffer) buffer = new unsigned char[bufSize]; if (!bufPosIn) bufPosIn = readUnbuffered(buffer, bufSize); - + /* Copy out the data in the buffer. */ size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; memcpy(data, buffer + bufPosOut, n); @@ -119,12 +129,19 @@ size_t FdSource::readUnbuffered(unsigned char * data, size_t len) checkInterrupt(); n = ::read(fd, (char *) data, bufSize); } while (n == -1 && errno == EINTR); - if (n == -1) throw SysError("reading from file"); - if (n == 0) throw EndOfFile("unexpected end-of-file"); + if (n == -1) { _good = false; throw SysError("reading from file"); } + if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } + read += n; return n; } +bool FdSource::good() +{ + return _good; +} + + size_t StringSource::read(unsigned char * data, size_t len) { if (pos == s.size()) throw EndOfFile("end of string reached"); @@ -144,56 +161,39 @@ void writePadding(size_t len, Sink & sink) } -void writeInt(unsigned int n, Sink & sink) -{ - unsigned char buf[8]; - memset(buf, 0, sizeof(buf)); - buf[0] = n & 0xff; - buf[1] = (n >> 8) & 0xff; - buf[2] = (n >> 16) & 0xff; - buf[3] = (n >> 24) & 0xff; - sink(buf, sizeof(buf)); -} - - -void writeLongLong(unsigned long long n, Sink & sink) -{ - unsigned char buf[8]; - buf[0] = n & 0xff; - buf[1] = (n >> 8) & 0xff; - buf[2] = (n >> 16) & 0xff; - buf[3] = (n >> 24) & 0xff; - buf[4] = (n >> 32) & 0xff; - buf[5] = (n >> 40) & 0xff; - buf[6] = (n >> 48) & 0xff; - buf[7] = (n >> 56) & 0xff; - sink(buf, sizeof(buf)); -} - - void writeString(const unsigned char * buf, size_t len, Sink & sink) { - writeInt(len, sink); + sink << len; sink(buf, len); writePadding(len, sink); } -void writeString(const string & s, Sink & sink) +Sink & operator << (Sink & sink, const string & s) { writeString((const unsigned char *) s.data(), s.size(), sink); + return sink; } template<class T> void writeStrings(const T & ss, Sink & sink) { - writeInt(ss.size(), sink); - foreach (typename T::const_iterator, i, ss) - writeString(*i, sink); + sink << ss.size(); + for (auto & i : ss) + sink << i; } -template void writeStrings(const Paths & ss, Sink & sink); -template void writeStrings(const PathSet & ss, Sink & sink); +Sink & operator << (Sink & sink, const Strings & s) +{ + writeStrings(s, sink); + return sink; +} + +Sink & operator << (Sink & sink, const StringSet & s) +{ + writeStrings(s, sink); + return sink; +} void readPadding(size_t len, Source & source) @@ -247,7 +247,7 @@ size_t readString(unsigned char * buf, size_t max, Source & source) return len; } - + string readString(Source & source) { size_t len = readInt(source); @@ -258,7 +258,20 @@ string readString(Source & source) return string((char *) buf, len); } - +Source & operator >> (Source & in, string & s) +{ + s = readString(in); + return in; +} + + +Source & operator >> (Source & in, unsigned int & n) +{ + n = readInt(in); + return in; +} + + template<class T> T readStrings(Source & source) { unsigned int count = readInt(source); @@ -275,11 +288,11 @@ template PathSet readStrings(Source & source); void StringSink::operator () (const unsigned char * data, size_t len) { static bool warned = false; - if (!warned && s.size() > threshold) { + if (!warned && s->size() > threshold) { warnLargeDump(); warned = true; } - s.append((const char *) data, len); + s->append((const char *) data, len); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 6a6f028aa652..892ec4aa36de 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -1,16 +1,23 @@ #pragma once #include "types.hh" +#include "util.hh" namespace nix { /* Abstract destination of binary data. */ -struct Sink +struct Sink { virtual ~Sink() { } virtual void operator () (const unsigned char * data, size_t len) = 0; + virtual bool good() { return true; } + + void operator () (const std::string & s) + { + (*this)((const unsigned char *) s.data(), s.size()); + } }; @@ -24,10 +31,15 @@ struct BufferedSink : Sink : bufSize(bufSize), bufPos(0), buffer(0) { } ~BufferedSink(); - void operator () (const unsigned char * data, size_t len); - + void operator () (const unsigned char * data, size_t len) override; + + void operator () (const std::string & s) + { + Sink::operator()(s); + } + void flush(); - + virtual void write(const unsigned char * data, size_t len) = 0; }; @@ -36,7 +48,7 @@ struct BufferedSink : Sink struct Source { virtual ~Source() { } - + /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. It blocks until all the requested data is available, or throws an error if it is not going to be available. */ @@ -46,6 +58,8 @@ struct Source return the number of bytes stored. If blocks until at least one byte is available. */ virtual size_t read(unsigned char * data, size_t len) = 0; + + virtual bool good() { return true; } }; @@ -58,9 +72,9 @@ struct BufferedSource : Source BufferedSource(size_t bufSize = 32 * 1024) : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } ~BufferedSource(); - - size_t read(unsigned char * data, size_t len); - + + size_t read(unsigned char * data, size_t len) override; + /* Underlying read call, to be overridden. */ virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; @@ -72,14 +86,19 @@ struct BufferedSource : Source struct FdSink : BufferedSink { int fd; - bool warn; - size_t written; + bool warn = false; + size_t written = 0; - FdSink() : fd(-1), warn(false), written(0) { } - FdSink(int fd) : fd(fd), warn(false), written(0) { } + FdSink() : fd(-1) { } + FdSink(int fd) : fd(fd) { } ~FdSink(); - - void write(const unsigned char * data, size_t len); + + void write(const unsigned char * data, size_t len) override; + + bool good() override; + +private: + bool _good = true; }; @@ -87,17 +106,24 @@ struct FdSink : BufferedSink struct FdSource : BufferedSource { int fd; + size_t read = 0; + FdSource() : fd(-1) { } FdSource(int fd) : fd(fd) { } - size_t readUnbuffered(unsigned char * data, size_t len); + size_t readUnbuffered(unsigned char * data, size_t len) override; + bool good() override; +private: + bool _good = true; }; /* A sink that writes data to a string. */ struct StringSink : Sink { - string s; - void operator () (const unsigned char * data, size_t len); + ref<std::string> s; + StringSink() : s(make_ref<std::string>()) { }; + StringSink(ref<std::string> s) : s(s) { }; + void operator () (const unsigned char * data, size_t len) override; }; @@ -107,16 +133,32 @@ struct StringSource : Source const string & s; size_t pos; StringSource(const string & _s) : s(_s), pos(0) { } - size_t read(unsigned char * data, size_t len); + size_t read(unsigned char * data, size_t len) override; }; void writePadding(size_t len, Sink & sink); -void writeInt(unsigned int n, Sink & sink); -void writeLongLong(unsigned long long n, Sink & sink); void writeString(const unsigned char * buf, size_t len, Sink & sink); -void writeString(const string & s, Sink & sink); -template<class T> void writeStrings(const T & ss, Sink & sink); + +inline Sink & operator << (Sink & sink, uint64_t n) +{ + unsigned char buf[8]; + buf[0] = n & 0xff; + buf[1] = (n >> 8) & 0xff; + buf[2] = (n >> 16) & 0xff; + buf[3] = (n >> 24) & 0xff; + buf[4] = (n >> 32) & 0xff; + buf[5] = (n >> 40) & 0xff; + buf[6] = (n >> 48) & 0xff; + buf[7] = (n >> 56) & 0xff; + sink(buf, sizeof(buf)); + return sink; +} + +Sink & operator << (Sink & sink, const string & s); +Sink & operator << (Sink & sink, const Strings & s); +Sink & operator << (Sink & sink, const StringSet & s); + void readPadding(size_t len, Source & source); unsigned int readInt(Source & source); @@ -125,6 +167,9 @@ size_t readString(unsigned char * buf, size_t max, Source & source); string readString(Source & source); template<class T> T readStrings(Source & source); +Source & operator >> (Source & in, string & s); +Source & operator >> (Source & in, unsigned int & n); + MakeError(SerialisationError, Error) diff --git a/src/libutil/sha1.c b/src/libutil/sha1.c deleted file mode 100644 index d9d294d15540..000000000000 --- a/src/libutil/sha1.c +++ /dev/null @@ -1,369 +0,0 @@ -/* $Id$ */ - -/* sha.c - Implementation of the Secure Hash Algorithm - * - * Copyright (C) 1995, A.M. Kuchling - * - * Distribute and use freely; there are no restrictions on further - * dissemination and usage except those imposed by the laws of your - * country of residence. - * - * Adapted to pike and some cleanup by Niels Möller. - */ - -/* $Id$ */ - -/* SHA: NIST's Secure Hash Algorithm */ - -/* Based on SHA code originally posted to sci.crypt by Peter Gutmann - in message <30ajo5$oe8@ccu2.auckland.ac.nz>. - Modified to test for endianness on creation of SHA objects by AMK. - Also, the original specification of SHA was found to have a weakness - by NSA/NIST. This code implements the fixed version of SHA. -*/ - -/* Here's the first paragraph of Peter Gutmann's posting: - -The following is my SHA (FIPS 180) code updated to allow use of the "fixed" -SHA, thanks to Jim Gillogly and an anonymous contributor for the information on -what's changed in the new version. The fix is a simple change which involves -adding a single rotate in the initial expansion function. It is unknown -whether this is an optimal solution to the problem which was discovered in the -SHA or whether it's simply a bandaid which fixes the problem with a minimum of -effort (for example the reengineering of a great many Capstone chips). -*/ - -#include "sha1.h" - -#include <string.h> - -void sha_copy(struct SHA_CTX *dest, struct SHA_CTX *src) -{ - unsigned int i; - - dest->count_l=src->count_l; - dest->count_h=src->count_h; - for(i=0; i<SHA_DIGESTLEN; i++) - dest->digest[i]=src->digest[i]; - for(i=0; i < src->index; i++) - dest->block[i] = src->block[i]; - dest->index = src->index; -} - - -/* The SHA f()-functions. The f1 and f3 functions can be optimized to - save one boolean operation each - thanks to Rich Schroeppel, - rcs@cs.arizona.edu for discovering this */ - -/*#define f1(x,y,z) ( ( x & y ) | ( ~x & z ) ) // Rounds 0-19 */ -#define f1(x,y,z) ( z ^ ( x & ( y ^ z ) ) ) /* Rounds 0-19 */ -#define f2(x,y,z) ( x ^ y ^ z ) /* Rounds 20-39 */ -/*#define f3(x,y,z) ( ( x & y ) | ( x & z ) | ( y & z ) ) // Rounds 40-59 */ -#define f3(x,y,z) ( ( x & y ) | ( z & ( x | y ) ) ) /* Rounds 40-59 */ -#define f4(x,y,z) ( x ^ y ^ z ) /* Rounds 60-79 */ - -/* The SHA Mysterious Constants */ - -#define K1 0x5A827999L /* Rounds 0-19 */ -#define K2 0x6ED9EBA1L /* Rounds 20-39 */ -#define K3 0x8F1BBCDCL /* Rounds 40-59 */ -#define K4 0xCA62C1D6L /* Rounds 60-79 */ - -/* SHA initial values */ - -#define h0init 0x67452301L -#define h1init 0xEFCDAB89L -#define h2init 0x98BADCFEL -#define h3init 0x10325476L -#define h4init 0xC3D2E1F0L - -/* 32-bit rotate left - kludged with shifts */ - -#define ROTL(n,X) ( ( (X) << (n) ) | ( (X) >> ( 32 - (n) ) ) ) - -/* The initial expanding function. The hash function is defined over an - 80-word expanded input array W, where the first 16 are copies of the input - data, and the remaining 64 are defined by - - W[ i ] = W[ i - 16 ] ^ W[ i - 14 ] ^ W[ i - 8 ] ^ W[ i - 3 ] - - This implementation generates these values on the fly in a circular - buffer - thanks to Colin Plumb, colin@nyx10.cs.du.edu for this - optimization. - - The updated SHA changes the expanding function by adding a rotate of 1 - bit. Thanks to Jim Gillogly, jim@rand.org, and an anonymous contributor - for this information */ - -#define expand(W,i) ( W[ i & 15 ] = \ - ROTL( 1, ( W[ i & 15 ] ^ W[ (i - 14) & 15 ] ^ \ - W[ (i - 8) & 15 ] ^ W[ (i - 3) & 15 ] ) ) ) - - -/* The prototype SHA sub-round. The fundamental sub-round is: - - a' = e + ROTL( 5, a ) + f( b, c, d ) + k + data; - b' = a; - c' = ROTL( 30, b ); - d' = c; - e' = d; - - but this is implemented by unrolling the loop 5 times and renaming the - variables ( e, a, b, c, d ) = ( a', b', c', d', e' ) each iteration. - This code is then replicated 20 times for each of the 4 functions, using - the next 20 values from the W[] array each time */ - -#define subRound(a, b, c, d, e, f, k, data) \ - ( e += ROTL( 5, a ) + f( b, c, d ) + k + data, b = ROTL( 30, b ) ) - -/* Initialize the SHA values */ - -void SHA1_Init(struct SHA_CTX *ctx) -{ - /* Set the h-vars to their initial values */ - ctx->digest[ 0 ] = h0init; - ctx->digest[ 1 ] = h1init; - ctx->digest[ 2 ] = h2init; - ctx->digest[ 3 ] = h3init; - ctx->digest[ 4 ] = h4init; - - /* Initialize bit count */ - ctx->count_l = ctx->count_h = 0; - - /* Initialize buffer */ - ctx->index = 0; -} - -/* Perform the SHA transformation. Note that this code, like MD5, seems to - break some optimizing compilers due to the complexity of the expressions - and the size of the basic block. It may be necessary to split it into - sections, e.g. based on the four subrounds - - Note that this function destroys the data area */ - -static void sha_transform(struct SHA_CTX *ctx, uint32_t *data ) -{ - uint32_t A, B, C, D, E; /* Local vars */ - - /* Set up first buffer and local data buffer */ - A = ctx->digest[0]; - B = ctx->digest[1]; - C = ctx->digest[2]; - D = ctx->digest[3]; - E = ctx->digest[4]; - - /* Heavy mangling, in 4 sub-rounds of 20 interations each. */ - subRound( A, B, C, D, E, f1, K1, data[ 0] ); - subRound( E, A, B, C, D, f1, K1, data[ 1] ); - subRound( D, E, A, B, C, f1, K1, data[ 2] ); - subRound( C, D, E, A, B, f1, K1, data[ 3] ); - subRound( B, C, D, E, A, f1, K1, data[ 4] ); - subRound( A, B, C, D, E, f1, K1, data[ 5] ); - subRound( E, A, B, C, D, f1, K1, data[ 6] ); - subRound( D, E, A, B, C, f1, K1, data[ 7] ); - subRound( C, D, E, A, B, f1, K1, data[ 8] ); - subRound( B, C, D, E, A, f1, K1, data[ 9] ); - subRound( A, B, C, D, E, f1, K1, data[10] ); - subRound( E, A, B, C, D, f1, K1, data[11] ); - subRound( D, E, A, B, C, f1, K1, data[12] ); - subRound( C, D, E, A, B, f1, K1, data[13] ); - subRound( B, C, D, E, A, f1, K1, data[14] ); - subRound( A, B, C, D, E, f1, K1, data[15] ); - subRound( E, A, B, C, D, f1, K1, expand( data, 16 ) ); - subRound( D, E, A, B, C, f1, K1, expand( data, 17 ) ); - subRound( C, D, E, A, B, f1, K1, expand( data, 18 ) ); - subRound( B, C, D, E, A, f1, K1, expand( data, 19 ) ); - - subRound( A, B, C, D, E, f2, K2, expand( data, 20 ) ); - subRound( E, A, B, C, D, f2, K2, expand( data, 21 ) ); - subRound( D, E, A, B, C, f2, K2, expand( data, 22 ) ); - subRound( C, D, E, A, B, f2, K2, expand( data, 23 ) ); - subRound( B, C, D, E, A, f2, K2, expand( data, 24 ) ); - subRound( A, B, C, D, E, f2, K2, expand( data, 25 ) ); - subRound( E, A, B, C, D, f2, K2, expand( data, 26 ) ); - subRound( D, E, A, B, C, f2, K2, expand( data, 27 ) ); - subRound( C, D, E, A, B, f2, K2, expand( data, 28 ) ); - subRound( B, C, D, E, A, f2, K2, expand( data, 29 ) ); - subRound( A, B, C, D, E, f2, K2, expand( data, 30 ) ); - subRound( E, A, B, C, D, f2, K2, expand( data, 31 ) ); - subRound( D, E, A, B, C, f2, K2, expand( data, 32 ) ); - subRound( C, D, E, A, B, f2, K2, expand( data, 33 ) ); - subRound( B, C, D, E, A, f2, K2, expand( data, 34 ) ); - subRound( A, B, C, D, E, f2, K2, expand( data, 35 ) ); - subRound( E, A, B, C, D, f2, K2, expand( data, 36 ) ); - subRound( D, E, A, B, C, f2, K2, expand( data, 37 ) ); - subRound( C, D, E, A, B, f2, K2, expand( data, 38 ) ); - subRound( B, C, D, E, A, f2, K2, expand( data, 39 ) ); - - subRound( A, B, C, D, E, f3, K3, expand( data, 40 ) ); - subRound( E, A, B, C, D, f3, K3, expand( data, 41 ) ); - subRound( D, E, A, B, C, f3, K3, expand( data, 42 ) ); - subRound( C, D, E, A, B, f3, K3, expand( data, 43 ) ); - subRound( B, C, D, E, A, f3, K3, expand( data, 44 ) ); - subRound( A, B, C, D, E, f3, K3, expand( data, 45 ) ); - subRound( E, A, B, C, D, f3, K3, expand( data, 46 ) ); - subRound( D, E, A, B, C, f3, K3, expand( data, 47 ) ); - subRound( C, D, E, A, B, f3, K3, expand( data, 48 ) ); - subRound( B, C, D, E, A, f3, K3, expand( data, 49 ) ); - subRound( A, B, C, D, E, f3, K3, expand( data, 50 ) ); - subRound( E, A, B, C, D, f3, K3, expand( data, 51 ) ); - subRound( D, E, A, B, C, f3, K3, expand( data, 52 ) ); - subRound( C, D, E, A, B, f3, K3, expand( data, 53 ) ); - subRound( B, C, D, E, A, f3, K3, expand( data, 54 ) ); - subRound( A, B, C, D, E, f3, K3, expand( data, 55 ) ); - subRound( E, A, B, C, D, f3, K3, expand( data, 56 ) ); - subRound( D, E, A, B, C, f3, K3, expand( data, 57 ) ); - subRound( C, D, E, A, B, f3, K3, expand( data, 58 ) ); - subRound( B, C, D, E, A, f3, K3, expand( data, 59 ) ); - - subRound( A, B, C, D, E, f4, K4, expand( data, 60 ) ); - subRound( E, A, B, C, D, f4, K4, expand( data, 61 ) ); - subRound( D, E, A, B, C, f4, K4, expand( data, 62 ) ); - subRound( C, D, E, A, B, f4, K4, expand( data, 63 ) ); - subRound( B, C, D, E, A, f4, K4, expand( data, 64 ) ); - subRound( A, B, C, D, E, f4, K4, expand( data, 65 ) ); - subRound( E, A, B, C, D, f4, K4, expand( data, 66 ) ); - subRound( D, E, A, B, C, f4, K4, expand( data, 67 ) ); - subRound( C, D, E, A, B, f4, K4, expand( data, 68 ) ); - subRound( B, C, D, E, A, f4, K4, expand( data, 69 ) ); - subRound( A, B, C, D, E, f4, K4, expand( data, 70 ) ); - subRound( E, A, B, C, D, f4, K4, expand( data, 71 ) ); - subRound( D, E, A, B, C, f4, K4, expand( data, 72 ) ); - subRound( C, D, E, A, B, f4, K4, expand( data, 73 ) ); - subRound( B, C, D, E, A, f4, K4, expand( data, 74 ) ); - subRound( A, B, C, D, E, f4, K4, expand( data, 75 ) ); - subRound( E, A, B, C, D, f4, K4, expand( data, 76 ) ); - subRound( D, E, A, B, C, f4, K4, expand( data, 77 ) ); - subRound( C, D, E, A, B, f4, K4, expand( data, 78 ) ); - subRound( B, C, D, E, A, f4, K4, expand( data, 79 ) ); - - /* Build message digest */ - ctx->digest[0] += A; - ctx->digest[1] += B; - ctx->digest[2] += C; - ctx->digest[3] += D; - ctx->digest[4] += E; -} - -#if 1 - -#ifndef EXTRACT_UCHAR -#define EXTRACT_UCHAR(p) (*(unsigned char *)(p)) -#endif - -#define STRING2INT(s) ((((((EXTRACT_UCHAR(s) << 8) \ - | EXTRACT_UCHAR(s+1)) << 8) \ - | EXTRACT_UCHAR(s+2)) << 8) \ - | EXTRACT_UCHAR(s+3)) -#else -uint32_t STRING2INT(unsigned char *s) -{ - uint32_t r; - unsigned int i; - - for (i = 0, r = 0; i < 4; i++, s++) - r = (r << 8) | *s; - return r; -} -#endif - -static void sha_block(struct SHA_CTX *ctx, const unsigned char *block) -{ - uint32_t data[SHA_DATALEN]; - unsigned int i; - - /* Update block count */ - if (!++ctx->count_l) - ++ctx->count_h; - - /* Endian independent conversion */ - for (i = 0; i<SHA_DATALEN; i++, block += 4) - data[i] = STRING2INT(block); - - sha_transform(ctx, data); -} - -void SHA1_Update(struct SHA_CTX *ctx, const unsigned char *buffer, uint32_t len) -{ - if (ctx->index) - { /* Try to fill partial block */ - unsigned left = SHA_DATASIZE - ctx->index; - if (len < left) - { - memcpy(ctx->block + ctx->index, buffer, len); - ctx->index += len; - return; /* Finished */ - } - else - { - memcpy(ctx->block + ctx->index, buffer, left); - sha_block(ctx, ctx->block); - buffer += left; - len -= left; - } - } - while (len >= SHA_DATASIZE) - { - sha_block(ctx, buffer); - buffer += SHA_DATASIZE; - len -= SHA_DATASIZE; - } - if ((ctx->index = len)) /* This assignment is intended */ - /* Buffer leftovers */ - memcpy(ctx->block, buffer, len); -} - -/* Final wrapup - pad to SHA_DATASIZE-byte boundary with the bit pattern - 1 0* (64-bit count of bits processed, MSB-first) */ - -void SHA1_Final(unsigned char *s, struct SHA_CTX *ctx) -{ - uint32_t data[SHA_DATALEN]; - unsigned int i; - unsigned int words; - - i = ctx->index; - /* Set the first char of padding to 0x80. This is safe since there is - always at least one byte free */ - ctx->block[i++] = 0x80; - - /* Fill rest of word */ - for( ; i & 3; i++) - ctx->block[i] = 0; - - /* i is now a multiple of the word size 4 */ - words = i >> 2; - for (i = 0; i < words; i++) - data[i] = STRING2INT(ctx->block + 4*i); - - if (words > (SHA_DATALEN-2)) - { /* No room for length in this block. Process it and - * pad with another one */ - for (i = words ; i < SHA_DATALEN; i++) - data[i] = 0; - sha_transform(ctx, data); - for (i = 0; i < (SHA_DATALEN-2); i++) - data[i] = 0; - } - else - for (i = words ; i < SHA_DATALEN - 2; i++) - data[i] = 0; - /* Theres 512 = 2^9 bits in one block */ - data[SHA_DATALEN-2] = (ctx->count_h << 9) | (ctx->count_l >> 23); - data[SHA_DATALEN-1] = (ctx->count_l << 9) | (ctx->index << 3); - sha_transform(ctx, data); - sha_digest(ctx, s); -} - -void sha_digest(struct SHA_CTX *ctx, unsigned char *s) -{ - unsigned int i; - - for (i = 0; i < SHA_DIGESTLEN; i++) - { - *s++ = ctx->digest[i] >> 24; - *s++ = 0xff & (ctx->digest[i] >> 16); - *s++ = 0xff & (ctx->digest[i] >> 8); - *s++ = 0xff & ctx->digest[i]; - } -} diff --git a/src/libutil/sha1.h b/src/libutil/sha1.h deleted file mode 100644 index 715040dd48df..000000000000 --- a/src/libutil/sha1.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef _SHA_H -#define _SHA_H - -#include <inttypes.h> - -/* The SHA block size and message digest sizes, in bytes */ - -#define SHA_DATASIZE 64 -#define SHA_DATALEN 16 -#define SHA_DIGESTSIZE 20 -#define SHA_DIGESTLEN 5 -/* The structure for storing SHA info */ - -struct SHA_CTX { - uint32_t digest[SHA_DIGESTLEN]; /* Message digest */ - uint32_t count_l, count_h; /* 64-bit block count */ - uint8_t block[SHA_DATASIZE]; /* SHA data buffer */ - unsigned int index; /* index into buffer */ -}; - -void SHA1_Init(struct SHA_CTX *ctx); -void SHA1_Update(struct SHA_CTX *ctx, const unsigned char *buffer, uint32_t len); -void SHA1_Final(unsigned char *s, struct SHA_CTX *ctx); -void sha_digest(struct SHA_CTX *ctx, unsigned char *s); -void sha_copy(struct SHA_CTX *dest, struct SHA_CTX *src); - - -#endif /* !_SHA_H */ diff --git a/src/libutil/sha256.c b/src/libutil/sha256.c deleted file mode 100644 index 63ed0ba43011..000000000000 --- a/src/libutil/sha256.c +++ /dev/null @@ -1,238 +0,0 @@ -/* crypto/sha/sha256.c */ -/* ==================================================================== - * Copyright (c) 2004 The OpenSSL Project. All rights reserved - * according to the OpenSSL license [found in ./md32_common.h]. - * ==================================================================== - */ - -#include <stdlib.h> -#include <string.h> - -#include "sha256.h" - -int SHA224_Init (SHA256_CTX *c) - { - c->h[0]=0xc1059ed8UL; c->h[1]=0x367cd507UL; - c->h[2]=0x3070dd17UL; c->h[3]=0xf70e5939UL; - c->h[4]=0xffc00b31UL; c->h[5]=0x68581511UL; - c->h[6]=0x64f98fa7UL; c->h[7]=0xbefa4fa4UL; - c->Nl=0; c->Nh=0; - c->num=0; c->md_len=SHA224_DIGEST_LENGTH; - return 1; - } - -int SHA256_Init (SHA256_CTX *c) - { - c->h[0]=0x6a09e667UL; c->h[1]=0xbb67ae85UL; - c->h[2]=0x3c6ef372UL; c->h[3]=0xa54ff53aUL; - c->h[4]=0x510e527fUL; c->h[5]=0x9b05688cUL; - c->h[6]=0x1f83d9abUL; c->h[7]=0x5be0cd19UL; - c->Nl=0; c->Nh=0; - c->num=0; c->md_len=SHA256_DIGEST_LENGTH; - return 1; - } - -unsigned char *SHA224(const unsigned char *d, size_t n, unsigned char *md) - { - SHA256_CTX c; - static unsigned char m[SHA224_DIGEST_LENGTH]; - - if (md == NULL) md=m; - SHA224_Init(&c); - SHA256_Update(&c,d,n); - SHA256_Final(md,&c); - return(md); - } - -unsigned char *SHA256(const unsigned char *d, size_t n, unsigned char *md) - { - SHA256_CTX c; - static unsigned char m[SHA256_DIGEST_LENGTH]; - - if (md == NULL) md=m; - SHA256_Init(&c); - SHA256_Update(&c,d,n); - SHA256_Final(md,&c); - return(md); - } - -int SHA224_Update(SHA256_CTX *c, const void *data, size_t len) -{ return SHA256_Update (c,data,len); } -int SHA224_Final (unsigned char *md, SHA256_CTX *c) -{ return SHA256_Final (md,c); } - -#define DATA_ORDER_IS_BIG_ENDIAN - -#define HASH_LONG uint32_t -#define HASH_LONG_LOG2 2 -#define HASH_CTX SHA256_CTX -#define HASH_CBLOCK SHA_CBLOCK -#define HASH_LBLOCK SHA_LBLOCK -/* - * Note that FIPS180-2 discusses "Truncation of the Hash Function Output." - * default: case below covers for it. It's not clear however if it's - * permitted to truncate to amount of bytes not divisible by 4. I bet not, - * but if it is, then default: case shall be extended. For reference. - * Idea behind separate cases for pre-defined lenghts is to let the - * compiler decide if it's appropriate to unroll small loops. - */ -#define HASH_MAKE_STRING(c,s) do { \ - unsigned long ll; \ - unsigned int n; \ - switch ((c)->md_len) \ - { case SHA224_DIGEST_LENGTH: \ - for (n=0;n<SHA224_DIGEST_LENGTH/4;n++) \ - { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ - break; \ - case SHA256_DIGEST_LENGTH: \ - for (n=0;n<SHA256_DIGEST_LENGTH/4;n++) \ - { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ - break; \ - default: \ - if ((c)->md_len > SHA256_DIGEST_LENGTH) \ - return 0; \ - for (n=0;n<(c)->md_len/4;n++) \ - { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ - break; \ - } \ - } while (0) - -#define HASH_UPDATE SHA256_Update -#define HASH_TRANSFORM SHA256_Transform -#define HASH_FINAL SHA256_Final -#define HASH_BLOCK_HOST_ORDER sha256_block_host_order -#define HASH_BLOCK_DATA_ORDER sha256_block_data_order -void sha256_block_host_order (SHA256_CTX *ctx, const void *in, size_t num); -void sha256_block_data_order (SHA256_CTX *ctx, const void *in, size_t num); - -#include "md32_common.h" - -static const uint32_t K256[64] = { - 0x428a2f98UL,0x71374491UL,0xb5c0fbcfUL,0xe9b5dba5UL, - 0x3956c25bUL,0x59f111f1UL,0x923f82a4UL,0xab1c5ed5UL, - 0xd807aa98UL,0x12835b01UL,0x243185beUL,0x550c7dc3UL, - 0x72be5d74UL,0x80deb1feUL,0x9bdc06a7UL,0xc19bf174UL, - 0xe49b69c1UL,0xefbe4786UL,0x0fc19dc6UL,0x240ca1ccUL, - 0x2de92c6fUL,0x4a7484aaUL,0x5cb0a9dcUL,0x76f988daUL, - 0x983e5152UL,0xa831c66dUL,0xb00327c8UL,0xbf597fc7UL, - 0xc6e00bf3UL,0xd5a79147UL,0x06ca6351UL,0x14292967UL, - 0x27b70a85UL,0x2e1b2138UL,0x4d2c6dfcUL,0x53380d13UL, - 0x650a7354UL,0x766a0abbUL,0x81c2c92eUL,0x92722c85UL, - 0xa2bfe8a1UL,0xa81a664bUL,0xc24b8b70UL,0xc76c51a3UL, - 0xd192e819UL,0xd6990624UL,0xf40e3585UL,0x106aa070UL, - 0x19a4c116UL,0x1e376c08UL,0x2748774cUL,0x34b0bcb5UL, - 0x391c0cb3UL,0x4ed8aa4aUL,0x5b9cca4fUL,0x682e6ff3UL, - 0x748f82eeUL,0x78a5636fUL,0x84c87814UL,0x8cc70208UL, - 0x90befffaUL,0xa4506cebUL,0xbef9a3f7UL,0xc67178f2UL }; - -/* - * FIPS specification refers to right rotations, while our ROTATE macro - * is left one. This is why you might notice that rotation coefficients - * differ from those observed in FIPS document by 32-N... - */ -#define Sigma0(x) (ROTATE((x),30) ^ ROTATE((x),19) ^ ROTATE((x),10)) -#define Sigma1(x) (ROTATE((x),26) ^ ROTATE((x),21) ^ ROTATE((x),7)) -#define sigma0(x) (ROTATE((x),25) ^ ROTATE((x),14) ^ ((x)>>3)) -#define sigma1(x) (ROTATE((x),15) ^ ROTATE((x),13) ^ ((x)>>10)) - -#define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) -#define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) - -#define ROUND_00_15(i,a,b,c,d,e,f,g,h) do { \ - T1 += h + Sigma1(e) + Ch(e,f,g) + K256[i]; \ - h = Sigma0(a) + Maj(a,b,c); \ - d += T1; h += T1; } while (0) - -#define ROUND_16_63(i,a,b,c,d,e,f,g,h,X) do { \ - s0 = X[(i+1)&0x0f]; s0 = sigma0(s0); \ - s1 = X[(i+14)&0x0f]; s1 = sigma1(s1); \ - T1 = X[(i)&0x0f] += s0 + s1 + X[(i+9)&0x0f]; \ - ROUND_00_15(i,a,b,c,d,e,f,g,h); } while (0) - -static void sha256_block (SHA256_CTX *ctx, const void *in, size_t num, int host) - { - uint32_t a,b,c,d,e,f,g,h,s0,s1,T1; - uint32_t X[16]; - int i; - const unsigned char *data=in; - - while (num--) { - - a = ctx->h[0]; b = ctx->h[1]; c = ctx->h[2]; d = ctx->h[3]; - e = ctx->h[4]; f = ctx->h[5]; g = ctx->h[6]; h = ctx->h[7]; - - if (host) - { - const uint32_t *W=(const uint32_t *)data; - - T1 = X[0] = W[0]; ROUND_00_15(0,a,b,c,d,e,f,g,h); - T1 = X[1] = W[1]; ROUND_00_15(1,h,a,b,c,d,e,f,g); - T1 = X[2] = W[2]; ROUND_00_15(2,g,h,a,b,c,d,e,f); - T1 = X[3] = W[3]; ROUND_00_15(3,f,g,h,a,b,c,d,e); - T1 = X[4] = W[4]; ROUND_00_15(4,e,f,g,h,a,b,c,d); - T1 = X[5] = W[5]; ROUND_00_15(5,d,e,f,g,h,a,b,c); - T1 = X[6] = W[6]; ROUND_00_15(6,c,d,e,f,g,h,a,b); - T1 = X[7] = W[7]; ROUND_00_15(7,b,c,d,e,f,g,h,a); - T1 = X[8] = W[8]; ROUND_00_15(8,a,b,c,d,e,f,g,h); - T1 = X[9] = W[9]; ROUND_00_15(9,h,a,b,c,d,e,f,g); - T1 = X[10] = W[10]; ROUND_00_15(10,g,h,a,b,c,d,e,f); - T1 = X[11] = W[11]; ROUND_00_15(11,f,g,h,a,b,c,d,e); - T1 = X[12] = W[12]; ROUND_00_15(12,e,f,g,h,a,b,c,d); - T1 = X[13] = W[13]; ROUND_00_15(13,d,e,f,g,h,a,b,c); - T1 = X[14] = W[14]; ROUND_00_15(14,c,d,e,f,g,h,a,b); - T1 = X[15] = W[15]; ROUND_00_15(15,b,c,d,e,f,g,h,a); - - data += SHA256_CBLOCK; - } - else - { - uint32_t l; - - HOST_c2l(data,l); T1 = X[0] = l; ROUND_00_15(0,a,b,c,d,e,f,g,h); - HOST_c2l(data,l); T1 = X[1] = l; ROUND_00_15(1,h,a,b,c,d,e,f,g); - HOST_c2l(data,l); T1 = X[2] = l; ROUND_00_15(2,g,h,a,b,c,d,e,f); - HOST_c2l(data,l); T1 = X[3] = l; ROUND_00_15(3,f,g,h,a,b,c,d,e); - HOST_c2l(data,l); T1 = X[4] = l; ROUND_00_15(4,e,f,g,h,a,b,c,d); - HOST_c2l(data,l); T1 = X[5] = l; ROUND_00_15(5,d,e,f,g,h,a,b,c); - HOST_c2l(data,l); T1 = X[6] = l; ROUND_00_15(6,c,d,e,f,g,h,a,b); - HOST_c2l(data,l); T1 = X[7] = l; ROUND_00_15(7,b,c,d,e,f,g,h,a); - HOST_c2l(data,l); T1 = X[8] = l; ROUND_00_15(8,a,b,c,d,e,f,g,h); - HOST_c2l(data,l); T1 = X[9] = l; ROUND_00_15(9,h,a,b,c,d,e,f,g); - HOST_c2l(data,l); T1 = X[10] = l; ROUND_00_15(10,g,h,a,b,c,d,e,f); - HOST_c2l(data,l); T1 = X[11] = l; ROUND_00_15(11,f,g,h,a,b,c,d,e); - HOST_c2l(data,l); T1 = X[12] = l; ROUND_00_15(12,e,f,g,h,a,b,c,d); - HOST_c2l(data,l); T1 = X[13] = l; ROUND_00_15(13,d,e,f,g,h,a,b,c); - HOST_c2l(data,l); T1 = X[14] = l; ROUND_00_15(14,c,d,e,f,g,h,a,b); - HOST_c2l(data,l); T1 = X[15] = l; ROUND_00_15(15,b,c,d,e,f,g,h,a); - } - - for (i=16;i<64;i+=8) - { - ROUND_16_63(i+0,a,b,c,d,e,f,g,h,X); - ROUND_16_63(i+1,h,a,b,c,d,e,f,g,X); - ROUND_16_63(i+2,g,h,a,b,c,d,e,f,X); - ROUND_16_63(i+3,f,g,h,a,b,c,d,e,X); - ROUND_16_63(i+4,e,f,g,h,a,b,c,d,X); - ROUND_16_63(i+5,d,e,f,g,h,a,b,c,X); - ROUND_16_63(i+6,c,d,e,f,g,h,a,b,X); - ROUND_16_63(i+7,b,c,d,e,f,g,h,a,X); - } - - ctx->h[0] += a; ctx->h[1] += b; ctx->h[2] += c; ctx->h[3] += d; - ctx->h[4] += e; ctx->h[5] += f; ctx->h[6] += g; ctx->h[7] += h; - - } - } - -/* - * Idea is to trade couple of cycles for some space. On IA-32 we save - * about 4K in "big footprint" case. In "small footprint" case any gain - * is appreciated:-) - */ -void HASH_BLOCK_HOST_ORDER (SHA256_CTX *ctx, const void *in, size_t num) -{ sha256_block (ctx,in,num,1); } - -void HASH_BLOCK_DATA_ORDER (SHA256_CTX *ctx, const void *in, size_t num) -{ sha256_block (ctx,in,num,0); } - - diff --git a/src/libutil/sha256.h b/src/libutil/sha256.h deleted file mode 100644 index 0686b84f0e08..000000000000 --- a/src/libutil/sha256.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef _SHA256_H -#define _SHA256_H 1 - -#include <inttypes.h> - -#define SHA_LBLOCK 16 -#define SHA_CBLOCK (SHA_LBLOCK*4) /* SHA treats input data as a - * contiguous array of 32 bit - * wide big-endian values. */ - -#define SHA256_CBLOCK (SHA_LBLOCK*4) /* SHA-256 treats input data as a - * contiguous array of 32 bit - * wide big-endian values. */ -#define SHA224_DIGEST_LENGTH 28 -#define SHA256_DIGEST_LENGTH 32 - -typedef struct SHA256state_st - { - uint32_t h[8]; - uint32_t Nl,Nh; - uint32_t data[SHA_LBLOCK]; - unsigned int num,md_len; - } SHA256_CTX; - -int SHA224_Init(SHA256_CTX *c); -int SHA224_Update(SHA256_CTX *c, const void *data, size_t len); -int SHA224_Final(unsigned char *md, SHA256_CTX *c); -unsigned char *SHA224(const unsigned char *d, size_t n,unsigned char *md); -int SHA256_Init(SHA256_CTX *c); -int SHA256_Update(SHA256_CTX *c, const void *data, size_t len); -int SHA256_Final(unsigned char *md, SHA256_CTX *c); -unsigned char *SHA256(const unsigned char *d, size_t n,unsigned char *md); -void SHA256_Transform(SHA256_CTX *c, const unsigned char *data); - -#endif diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh new file mode 100644 index 000000000000..ebe64ffbdab7 --- /dev/null +++ b/src/libutil/sync.hh @@ -0,0 +1,78 @@ +#pragma once + +#include <mutex> +#include <condition_variable> +#include <cassert> + +namespace nix { + +/* This template class ensures synchronized access to a value of type + T. It is used as follows: + + struct Data { int x; ... }; + + Sync<Data> data; + + { + auto data_(data.lock()); + data_->x = 123; + } + + Here, "data" is automatically unlocked when "data_" goes out of + scope. +*/ + +template<class T, class M = std::mutex> +class Sync +{ +private: + M mutex; + T data; + +public: + + Sync() { } + Sync(const T & data) : data(data) { } + + class Lock + { + private: + Sync * s; + std::unique_lock<M> lk; + friend Sync; + Lock(Sync * s) : s(s), lk(s->mutex) { } + public: + Lock(Lock && l) : s(l.s) { abort(); } + Lock(const Lock & l) = delete; + ~Lock() { } + T * operator -> () { return &s->data; } + T & operator * () { return s->data; } + + void wait(std::condition_variable & cv) + { + assert(s); + cv.wait(lk); + } + + template<class Rep, class Period, class Predicate> + bool wait_for(std::condition_variable & cv, + const std::chrono::duration<Rep, Period> & duration, + Predicate pred) + { + assert(s); + return cv.wait_for(lk, duration, pred); + } + + template<class Clock, class Duration> + std::cv_status wait_until(std::condition_variable & cv, + const std::chrono::time_point<Clock, Duration> & duration) + { + assert(s); + return cv.wait_until(lk, duration); + } + }; + + Lock lock() { return Lock(this); } +}; + +} diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc new file mode 100644 index 000000000000..32363ecf0098 --- /dev/null +++ b/src/libutil/thread-pool.cc @@ -0,0 +1,102 @@ +#include "thread-pool.hh" +#include "affinity.hh" + +namespace nix { + +ThreadPool::ThreadPool(size_t _maxThreads) + : maxThreads(_maxThreads) +{ + restoreAffinity(); // FIXME + + if (!maxThreads) { + maxThreads = std::thread::hardware_concurrency(); + if (!maxThreads) maxThreads = 1; + } + + debug(format("starting pool of %d threads") % maxThreads); +} + +ThreadPool::~ThreadPool() +{ + std::vector<std::thread> workers; + { + auto state(state_.lock()); + state->quit = true; + std::swap(workers, state->workers); + } + + debug(format("reaping %d worker threads") % workers.size()); + + work.notify_all(); + + for (auto & thr : workers) + thr.join(); +} + +void ThreadPool::enqueue(const work_t & t) +{ + auto state(state_.lock()); + assert(!state->quit); + state->left.push(t); + if (state->left.size() > state->workers.size() && state->workers.size() < maxThreads) + state->workers.emplace_back(&ThreadPool::workerEntry, this); + work.notify_one(); +} + +void ThreadPool::process() +{ + while (true) { + auto state(state_.lock()); + if (state->exception) + std::rethrow_exception(state->exception); + if (state->left.empty() && !state->pending) break; + state.wait(done); + } +} + +void ThreadPool::workerEntry() +{ + bool didWork = false; + + while (true) { + work_t w; + { + auto state(state_.lock()); + while (true) { + if (state->quit || state->exception) return; + if (didWork) { + assert(state->pending); + state->pending--; + didWork = false; + } + if (!state->left.empty()) break; + if (!state->pending) + done.notify_all(); + state.wait(work); + } + w = state->left.front(); + state->left.pop(); + state->pending++; + } + + try { + w(); + } catch (std::exception & e) { + auto state(state_.lock()); + if (state->exception) { + if (!dynamic_cast<Interrupted*>(&e)) + printMsg(lvlError, format("error: %s") % e.what()); + } else { + state->exception = std::current_exception(); + work.notify_all(); + done.notify_all(); + } + } + + didWork = true; + } +} + +} + + diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh new file mode 100644 index 000000000000..78b63467d62e --- /dev/null +++ b/src/libutil/thread-pool.hh @@ -0,0 +1,117 @@ +#pragma once + +#include "sync.hh" +#include "util.hh" + +#include <queue> +#include <functional> +#include <thread> +#include <map> + +namespace nix { + +/* A simple thread pool that executes a queue of work items + (lambdas). */ +class ThreadPool +{ +public: + + ThreadPool(size_t maxThreads = 0); + + ~ThreadPool(); + + // FIXME: use std::packaged_task? + typedef std::function<void()> work_t; + + /* Enqueue a function to be executed by the thread pool. */ + void enqueue(const work_t & t); + + /* Execute work items until the queue is empty. Note that work + items are allowed to add new items to the queue; this is + handled correctly. Queue processing stops prematurely if any + work item throws an exception. This exception is propagated to + the calling thread. If multiple work items throw an exception + concurrently, only one item is propagated; the others are + printed on stderr and otherwise ignored. */ + void process(); + +private: + + size_t maxThreads; + + struct State + { + std::queue<work_t> left; + size_t pending = 0; + std::exception_ptr exception; + std::vector<std::thread> workers; + bool quit = false; + }; + + Sync<State> state_; + + std::condition_variable work, done; + + void workerEntry(); +}; + +/* Process in parallel a set of items of type T that have a partial + ordering between them. Thus, any item is only processed after all + its dependencies have been processed. */ +template<typename T> +void processGraph( + ThreadPool & pool, + const std::set<T> & nodes, + std::function<std::set<T>(const T &)> getEdges, + std::function<void(const T &)> processNode) +{ + struct Graph { + std::set<T> left; + std::map<T, std::set<T>> refs, rrefs; + std::function<void(T)> wrap; + }; + + ref<Sync<Graph>> graph_ = make_ref<Sync<Graph>>(); + + auto wrapWork = [&pool, graph_, processNode](const T & node) { + processNode(node); + + /* Enqueue work for all nodes that were waiting on this one. */ + { + auto graph(graph_->lock()); + graph->left.erase(node); + for (auto & rref : graph->rrefs[node]) { + auto & refs(graph->refs[rref]); + auto i = refs.find(node); + assert(i != refs.end()); + refs.erase(i); + if (refs.empty()) + pool.enqueue(std::bind(graph->wrap, rref)); + } + } + }; + + { + auto graph(graph_->lock()); + graph->left = nodes; + graph->wrap = wrapWork; + } + + /* Build the dependency graph; enqueue all nodes with no + dependencies. */ + for (auto & node : nodes) { + auto refs = getEdges(node); + { + auto graph(graph_->lock()); + for (auto & ref : refs) + if (ref != node && graph->left.count(ref)) { + graph->refs[node].insert(ref); + graph->rrefs[ref].insert(node); + } + if (graph->refs[node].empty()) + pool.enqueue(std::bind(graph->wrap, node)); + } + } +} + +} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 160884ee1ad7..bd192b8506b2 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -2,9 +2,12 @@ #include "config.h" +#include "ref.hh" + #include <string> #include <list> #include <set> +#include <memory> #include <boost/format.hpp> @@ -86,14 +89,4 @@ typedef list<Path> Paths; typedef set<Path> PathSet; -typedef enum { - lvlError = 0, - lvlInfo, - lvlTalkative, - lvlChatty, - lvlDebug, - lvlVomit -} Verbosity; - - } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 596b79e10e69..67558cc0b33c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -149,10 +149,20 @@ Path dirOf(const Path & path) string baseNameOf(const Path & path) { - Path::size_type pos = path.rfind('/'); + if (path.empty()) + return ""; + + Path::size_type last = path.length() - 1; + if (path[last] == '/' && last > 0) + last -= 1; + + Path::size_type pos = path.rfind('/', last); if (pos == string::npos) - throw Error(format("invalid file name ‘%1%’") % path); - return string(path, pos + 1); + pos = 0; + else + pos += 1; + + return string(path, pos, last - pos + 1); } @@ -199,7 +209,7 @@ Path readLink(const Path & path) else if (rlsize > st.st_size) throw Error(format("symbolic link ‘%1%’ size overflow %2% > %3%") % path % rlsize % st.st_size); - return string(buf, st.st_size); + return string(buf, rlsize); } @@ -223,7 +233,13 @@ DirEntries readDirectory(const Path & path) checkInterrupt(); string name = dirent->d_name; if (name == "." || name == "..") continue; - entries.emplace_back(name, dirent->d_ino, dirent->d_type); + entries.emplace_back(name, dirent->d_ino, +#ifdef HAVE_STRUCT_DIRENT_D_TYPE + dirent->d_type +#else + DT_UNKNOWN +#endif + ); } if (errno) throw SysError(format("reading directory ‘%1%’") % path); @@ -304,9 +320,11 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) { checkInterrupt(); - printMsg(lvlVomit, format("%1%") % path); - - struct stat st = lstat(path); + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + if (errno == ENOENT) return; + throw SysError(format("getting status of ‘%1%’") % path); + } if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) bytesFreed += st.st_blocks * 512; @@ -322,8 +340,10 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) _deletePath(path + "/" + i.name, bytesFreed); } - if (remove(path.c_str()) == -1) + if (remove(path.c_str()) == -1) { + if (errno == ENOENT) return; throw SysError(format("cannot unlink ‘%1%’") % path); + } } @@ -336,8 +356,7 @@ void deletePath(const Path & path) void deletePath(const Path & path, unsigned long long & bytesFreed) { - startNest(nest, lvlDebug, - format("recursively deleting path ‘%1%’") % path); + Activity act(*logger, lvlDebug, format("recursively deleting path ‘%1%’") % path); bytesFreed = 0; _deletePath(path, bytesFreed); } @@ -383,6 +402,18 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, } +Path getCacheDir() +{ + Path cacheDir = getEnv("XDG_CACHE_HOME"); + if (cacheDir.empty()) { + Path homeDir = getEnv("HOME"); + if (homeDir.empty()) throw Error("$XDG_CACHE_HOME and $HOME are not set"); + cacheDir = homeDir + "/.cache"; + } + return cacheDir; +} + + Paths createDirs(const Path & path) { Paths created; @@ -424,101 +455,6 @@ void replaceSymlink(const Path & target, const Path & link) } -LogType logType = ltPretty; -Verbosity verbosity = lvlInfo; - -static int nestingLevel = 0; - - -Nest::Nest() -{ - nest = false; -} - - -Nest::~Nest() -{ - close(); -} - - -static string escVerbosity(Verbosity level) -{ - return int2String((int) level); -} - - -void Nest::open(Verbosity level, const FormatOrString & fs) -{ - if (level <= verbosity) { - if (logType == ltEscapes) - std::cerr << "\033[" << escVerbosity(level) << "p" - << fs.s << "\n"; - else - printMsg_(level, fs); - nest = true; - nestingLevel++; - } -} - - -void Nest::close() -{ - if (nest) { - nestingLevel--; - if (logType == ltEscapes) - std::cerr << "\033[q"; - nest = false; - } -} - - -void printMsg_(Verbosity level, const FormatOrString & fs) -{ - checkInterrupt(); - if (level > verbosity) return; - string prefix; - if (logType == ltPretty) - for (int i = 0; i < nestingLevel; i++) - prefix += "| "; - else if (logType == ltEscapes && level != lvlInfo) - prefix = "\033[" + escVerbosity(level) + "s"; - string s = (format("%1%%2%\n") % prefix % fs.s).str(); - if (!isatty(STDERR_FILENO)) s = filterANSIEscapes(s); - writeToStderr(s); -} - - -void warnOnce(bool & haveWarned, const FormatOrString & fs) -{ - if (!haveWarned) { - printMsg(lvlError, format("warning: %1%") % fs.s); - haveWarned = true; - } -} - - -void writeToStderr(const string & s) -{ - try { - if (_writeToStderr) - _writeToStderr((const unsigned char *) s.data(), s.size()); - else - writeFull(STDERR_FILENO, s); - } catch (SysError & e) { - /* Ignore failing writes to stderr if we're in an exception - handler, otherwise throw an exception. We need to ignore - write errors in exception handlers to ensure that cleanup - code runs to completion if the other side of stderr has - been closed unexpectedly. */ - if (!std::uncaught_exception()) throw; - } -} - - -void (*_writeToStderr) (const unsigned char * buf, size_t count) = 0; - - void readFull(int fd, unsigned char * buf, size_t count) { while (count) { @@ -578,6 +514,8 @@ string drainFD(int fd) ////////////////////////////////////////////////////////////////////// +AutoDelete::AutoDelete() : del{false} {} + AutoDelete::AutoDelete(const string & p, bool recursive) : path(p) { del = true; @@ -605,6 +543,12 @@ void AutoDelete::cancel() del = false; } +void AutoDelete::reset(const Path & p, bool recursive) { + path = p; + this->recursive = recursive; + del = true; +} + ////////////////////////////////////////////////////////////////////// @@ -901,7 +845,8 @@ static pid_t doFork(bool allowVfork, std::function<void()> fun) pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) { auto wrapper = [&]() { - if (!options.allowVfork) _writeToStderr = 0; + if (!options.allowVfork) + logger = makeDefaultLogger(); try { #if __linux__ if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) @@ -1022,13 +967,15 @@ void restoreSIGPIPE() volatile sig_atomic_t _isInterrupted = 0; +thread_local bool interruptThrown = false; + void _interrupted() { /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled kills the program! */ - if (!std::uncaught_exception()) { - _isInterrupted = 0; + if (!interruptThrown && !std::uncaught_exception()) { + interruptThrown = true; throw Interrupted("interrupted by the user"); } } @@ -1060,9 +1007,9 @@ template vector<string> tokenizeString(const string & s, const string & separato string concatStringsSep(const string & sep, const Strings & ss) { string s; - foreach (Strings::const_iterator, i, ss) { + for (auto & i : ss) { if (s.size() != 0) s += sep; - s += *i; + s += i; } return s; } @@ -1071,9 +1018,9 @@ string concatStringsSep(const string & sep, const Strings & ss) string concatStringsSep(const string & sep, const StringSet & ss) { string s; - foreach (StringSet::const_iterator, i, ss) { + for (auto & i : ss) { if (s.size() != 0) s += sep; - s += *i; + s += i; } return s; } @@ -1135,6 +1082,12 @@ bool statusOk(int status) } +bool hasPrefix(const string & s, const string & suffix) +{ + return s.compare(0, suffix.size(), suffix) == 0; +} + + bool hasSuffix(const string & s, const string & suffix) { return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 187e05ece050..ab43637a574c 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -1,6 +1,7 @@ #pragma once #include "types.hh" +#include "logging.hh" #include <sys/types.h> #include <sys/stat.h> @@ -8,20 +9,19 @@ #include <unistd.h> #include <signal.h> #include <functional> - +#include <limits> #include <cstdio> +#ifndef HAVE_STRUCT_DIRENT_D_TYPE +#define DT_UNKNOWN 0 +#define DT_REG 1 +#define DT_LNK 2 +#define DT_DIR 3 +#endif namespace nix { -#define foreach(it_type, it, collection) \ - for (it_type it = (collection).begin(); it != (collection).end(); ++it) - -#define foreach_reverse(it_type, it, collection) \ - for (it_type it = (collection).rbegin(); it != (collection).rend(); ++it) - - /* Return an environment variable. */ string getEnv(const string & key, const string & def = ""); @@ -93,8 +93,8 @@ string readLine(int fd); void writeLine(int fd, string s); /* Delete a path; i.e., in the case of a directory, it is deleted - recursively. Don't use this at home, kids. The second variant - returns the number of bytes and blocks freed. */ + recursively. It's not an error if the path does not exist. The + second variant returns the number of bytes and blocks freed. */ void deletePath(const Path & path); void deletePath(const Path & path, unsigned long long & bytesFreed); @@ -103,6 +103,9 @@ void deletePath(const Path & path, unsigned long long & bytesFreed); Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); +/* Return the path to $XDG_CACHE_HOME/.cache. */ +Path getCacheDir(); + /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */ Paths createDirs(const Path & path); @@ -114,62 +117,6 @@ void createSymlink(const Path & target, const Path & link); void replaceSymlink(const Path & target, const Path & link); -template<class T, class A> -T singleton(const A & a) -{ - T t; - t.insert(a); - return t; -} - - -/* Messages. */ - - -typedef enum { - ltPretty, /* nice, nested output */ - ltEscapes, /* nesting indicated using escape codes (for log2xml) */ - ltFlat /* no nesting */ -} LogType; - -extern LogType logType; -extern Verbosity verbosity; /* suppress msgs > this */ - -class Nest -{ -private: - bool nest; -public: - Nest(); - ~Nest(); - void open(Verbosity level, const FormatOrString & fs); - void close(); -}; - -void printMsg_(Verbosity level, const FormatOrString & fs); - -#define startNest(varName, level, f) \ - Nest varName; \ - if (level <= verbosity) { \ - varName.open(level, (f)); \ - } - -#define printMsg(level, f) \ - do { \ - if (level <= verbosity) { \ - printMsg_(level, (f)); \ - } \ - } while (0) - -#define debug(f) printMsg(lvlDebug, f) - -void warnOnce(bool & haveWarned, const FormatOrString & fs); - -void writeToStderr(const string & s); - -extern void (*_writeToStderr) (const unsigned char * buf, size_t count); - - /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ void readFull(int fd, unsigned char * buf, size_t count); @@ -205,9 +152,12 @@ class AutoDelete bool del; bool recursive; public: + AutoDelete(); AutoDelete(const Path & p, bool recursive = true); ~AutoDelete(); void cancel(); + void reset(const Path & p, bool recursive = true); + operator Path() const { return path; } }; @@ -313,6 +263,8 @@ void restoreSIGPIPE(); extern volatile sig_atomic_t _isInterrupted; +extern thread_local bool interruptThrown; + void _interrupted(); void inline checkInterrupt() @@ -356,19 +308,26 @@ bool statusOk(int status); /* Parse a string into an integer. */ template<class N> bool string2Int(const string & s, N & n) { + if (string(s, 0, 1) == "-" && !std::numeric_limits<N>::is_signed) + return false; std::istringstream str(s); str >> n; return str && str.get() == EOF; } -template<class N> string int2String(N n) +/* Parse a string into a float. */ +template<class N> bool string2Float(const string & s, N & n) { - std::ostringstream str; - str << n; - return str.str(); + std::istringstream str(s); + str >> n; + return str && str.get() == EOF; } +/* Return true iff `s' starts with `prefix'. */ +bool hasPrefix(const string & s, const string & prefix); + + /* Return true iff `s' ends in `suffix'. */ bool hasSuffix(const string & s, const string & suffix); @@ -415,4 +374,14 @@ string base64Encode(const string & s); string base64Decode(const string & s); +/* Get a value for the specified key from an associate container, or a + default value if the key doesn't exist. */ +template <class T> +string get(const T & map, const string & key, const string & def = "") +{ + auto i = map.find(key); + return i == map.end() ? def : i->second; +} + + } diff --git a/src/libutil/xml-writer.cc b/src/libutil/xml-writer.cc index 01794001b2c6..98bd058d18be 100644 --- a/src/libutil/xml-writer.cc +++ b/src/libutil/xml-writer.cc @@ -73,10 +73,10 @@ void XMLWriter::writeEmptyElement(const string & name, void XMLWriter::writeAttrs(const XMLAttrs & attrs) { - for (XMLAttrs::const_iterator i = attrs.begin(); i != attrs.end(); ++i) { - output << " " << i->first << "=\""; - for (unsigned int j = 0; j < i->second.size(); ++j) { - char c = i->second[j]; + for (auto & i : attrs) { + output << " " << i.first << "=\""; + for (unsigned int j = 0; j < i.second.size(); ++j) { + char c = i.second[j]; if (c == '"') output << """; else if (c == '<') output << "<"; else if (c == '>') output << ">"; |