diff options
Diffstat (limited to 'src/libutil')
| -rw-r--r-- | src/libutil/affinity.cc | 55 | ||||
| -rw-r--r-- | src/libutil/affinity.hh | 9 | ||||
| -rw-r--r-- | src/libutil/archive.cc | 335 | ||||
| -rw-r--r-- | src/libutil/archive.hh | 75 | ||||
| -rw-r--r-- | src/libutil/hash.cc | 382 | ||||
| -rw-r--r-- | src/libutil/hash.hh | 113 | ||||
| -rw-r--r-- | src/libutil/local.mk | 15 | ||||
| -rw-r--r-- | src/libutil/md32_common.h | 620 | ||||
| -rw-r--r-- | src/libutil/md5.c | 365 | ||||
| -rw-r--r-- | src/libutil/md5.h | 82 | ||||
| -rw-r--r-- | src/libutil/serialise.cc | 259 | ||||
| -rw-r--r-- | src/libutil/serialise.hh | 133 | ||||
| -rw-r--r-- | src/libutil/sha1.c | 369 | ||||
| -rw-r--r-- | src/libutil/sha1.h | 28 | ||||
| -rw-r--r-- | src/libutil/sha256.c | 238 | ||||
| -rw-r--r-- | src/libutil/sha256.h | 35 | ||||
| -rw-r--r-- | src/libutil/types.hh | 86 | ||||
| -rw-r--r-- | src/libutil/util.cc | 1105 | ||||
| -rw-r--r-- | src/libutil/util.hh | 349 | ||||
| -rw-r--r-- | src/libutil/xml-writer.cc | 94 | ||||
| -rw-r--r-- | src/libutil/xml-writer.hh | 69 |
21 files changed, 4816 insertions, 0 deletions
diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc new file mode 100644 index 000000000000..3e21f43a2e9d --- /dev/null +++ b/src/libutil/affinity.cc @@ -0,0 +1,55 @@ +#include "types.hh" +#include "util.hh" +#include "affinity.hh" + +#if HAVE_SCHED_H +#include <sched.h> +#endif + +namespace nix { + + +#if HAVE_SCHED_SETAFFINITY +static bool didSaveAffinity = false; +static cpu_set_t savedAffinity; +#endif + + +void setAffinityTo(int cpu) +{ +#if HAVE_SCHED_SETAFFINITY + if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; + didSaveAffinity = true; + printMsg(lvlDebug, format("locking this thread to CPU %1%") % cpu); + cpu_set_t newAffinity; + CPU_ZERO(&newAffinity); + CPU_SET(cpu, &newAffinity); + if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) + printMsg(lvlError, format("failed to lock thread to CPU %1%") % cpu); +#endif +} + + +int lockToCurrentCPU() +{ +#if HAVE_SCHED_SETAFFINITY + int cpu = sched_getcpu(); + if (cpu != -1) setAffinityTo(cpu); + return cpu; +#else + return -1; +#endif +} + + +void restoreAffinity() +{ +#if HAVE_SCHED_SETAFFINITY + if (!didSaveAffinity) return; + if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) + printMsg(lvlError, "failed to restore affinity %1%"); +#endif +} + + +} diff --git a/src/libutil/affinity.hh b/src/libutil/affinity.hh new file mode 100644 index 000000000000..c1bd28e1367a --- /dev/null +++ b/src/libutil/affinity.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix { + +void setAffinityTo(int cpu); +int lockToCurrentCPU(); +void restoreAffinity(); + +} diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc new file mode 100644 index 000000000000..70a1c580dd1b --- /dev/null +++ b/src/libutil/archive.cc @@ -0,0 +1,335 @@ +#include "config.h" + +#include <cerrno> +#include <algorithm> +#include <vector> + +#define _XOPEN_SOURCE 600 +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <dirent.h> +#include <fcntl.h> + +#include "archive.hh" +#include "util.hh" + + +namespace nix { + + +static string archiveVersion1 = "nix-archive-1"; + + +PathFilter defaultPathFilter; + + +static void dump(const string & path, Sink & sink, PathFilter & filter); + + +static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter) +{ + Strings names = readDirectory(path); + vector<string> names2(names.begin(), names.end()); + sort(names2.begin(), names2.end()); + + for (vector<string>::iterator i = names2.begin(); + i != names2.end(); ++i) + { + Path entry = path + "/" + *i; + if (filter(entry)) { + writeString("entry", sink); + writeString("(", sink); + writeString("name", sink); + writeString(*i, sink); + writeString("node", sink); + dump(entry, sink, filter); + writeString(")", sink); + } + } +} + + +static void dumpContents(const Path & path, size_t size, + Sink & sink) +{ + writeString("contents", sink); + writeLongLong(size, sink); + + AutoCloseFD fd = open(path.c_str(), O_RDONLY); + if (fd == -1) throw SysError(format("opening file `%1%'") % path); + + unsigned char buf[65536]; + size_t left = size; + + while (left > 0) { + size_t n = left > sizeof(buf) ? sizeof(buf) : left; + readFull(fd, buf, n); + left -= n; + sink(buf, n); + } + + writePadding(size, sink); +} + + +static void dump(const Path & path, Sink & sink, PathFilter & filter) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + + writeString("(", sink); + + if (S_ISREG(st.st_mode)) { + writeString("type", sink); + writeString("regular", sink); + if (st.st_mode & S_IXUSR) { + writeString("executable", sink); + writeString("", sink); + } + dumpContents(path, (size_t) st.st_size, sink); + } + + else if (S_ISDIR(st.st_mode)) { + writeString("type", sink); + writeString("directory", sink); + dumpEntries(path, sink, filter); + } + + else if (S_ISLNK(st.st_mode)) { + writeString("type", sink); + writeString("symlink", sink); + writeString("target", sink); + writeString(readLink(path), sink); + } + + else throw Error(format("file `%1%' has an unsupported type") % path); + + writeString(")", sink); +} + + +void dumpPath(const Path & path, Sink & sink, PathFilter & filter) +{ + writeString(archiveVersion1, sink); + dump(path, sink, filter); +} + + +static SerialisationError badArchive(string s) +{ + return SerialisationError("bad archive: " + s); +} + + +static void skipGeneric(Source & source) +{ + if (readString(source) == "(") { + while (readString(source) != ")") + skipGeneric(source); + } +} + + +static void parse(ParseSink & sink, Source & source, const Path & path); + + + +static void parseEntry(ParseSink & sink, Source & source, const Path & path) +{ + string s, name; + + s = readString(source); + if (s != "(") throw badArchive("expected open tag"); + + while (1) { + checkInterrupt(); + + s = readString(source); + + if (s == ")") { + break; + } else if (s == "name") { + name = readString(source); + } else if (s == "node") { + if (s == "") throw badArchive("entry name missing"); + parse(sink, source, path + "/" + name); + } else { + throw badArchive("unknown field " + s); + skipGeneric(source); + } + } +} + + +static void parseContents(ParseSink & sink, Source & source, const Path & path) +{ + unsigned long long size = readLongLong(source); + + sink.preallocateContents(size); + + unsigned long long left = size; + unsigned char buf[65536]; + + while (left) { + checkInterrupt(); + unsigned int n = sizeof(buf); + if ((unsigned long long) n > left) n = left; + source(buf, n); + sink.receiveContents(buf, n); + left -= n; + } + + readPadding(size, source); +} + + +static void parse(ParseSink & sink, Source & source, const Path & path) +{ + string s; + + s = readString(source); + if (s != "(") throw badArchive("expected open tag"); + + enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; + + while (1) { + checkInterrupt(); + + s = readString(source); + + if (s == ")") { + break; + } + + else if (s == "type") { + if (type != tpUnknown) + throw badArchive("multiple type fields"); + string t = readString(source); + + if (t == "regular") { + type = tpRegular; + sink.createRegularFile(path); + } + + else if (t == "directory") { + sink.createDirectory(path); + type = tpDirectory; + } + + else if (t == "symlink") { + type = tpSymlink; + } + + else throw badArchive("unknown file type " + t); + + } + + else if (s == "contents" && type == tpRegular) { + parseContents(sink, source, path); + } + + else if (s == "executable" && type == tpRegular) { + readString(source); + sink.isExecutable(); + } + + else if (s == "entry" && type == tpDirectory) { + parseEntry(sink, source, path); + } + + else if (s == "target" && type == tpSymlink) { + string target = readString(source); + sink.createSymlink(path, target); + } + + else { + throw badArchive("unknown field " + s); + skipGeneric(source); + } + } +} + + +void parseDump(ParseSink & sink, Source & source) +{ + string version; + try { + version = readString(source); + } catch (SerialisationError & e) { + /* This generally means the integer at the start couldn't be + decoded. Ignore and throw the exception below. */ + } + if (version != archiveVersion1) + throw badArchive("input doesn't look like a Nix archive"); + parse(sink, source, ""); +} + + +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + void createDirectory(const Path & path) + { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError(format("creating directory `%1%'") % p); + }; + + void createRegularFile(const Path & path) + { + Path p = dstPath + path; + fd.close(); + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0666); + if (fd == -1) throw SysError(format("creating file `%1%'") % p); + } + + void isExecutable() + { + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("fstat"); + if (fchmod(fd, st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } + + void preallocateContents(unsigned long long len) + { +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd, 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL) + throw SysError(format("preallocating file of %1% bytes") % len); + } +#endif + } + + void receiveContents(unsigned char * data, unsigned int len) + { + writeFull(fd, data, len); + } + + void createSymlink(const Path & path, const string & target) + { + Path p = dstPath + path; + nix::createSymlink(target, p); + } +}; + + +void restorePath(const Path & path, Source & source) +{ + RestoreSink sink; + sink.dstPath = path; + parseDump(sink, source); +} + + +} diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh new file mode 100644 index 000000000000..ccac92074d54 --- /dev/null +++ b/src/libutil/archive.hh @@ -0,0 +1,75 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" + + +namespace nix { + + +/* dumpPath creates a Nix archive of the specified path. The format + is as follows: + + IF path points to a REGULAR FILE: + dump(path) = attrs( + [ ("type", "regular") + , ("contents", contents(path)) + ]) + + IF path points to a DIRECTORY: + dump(path) = attrs( + [ ("type", "directory") + , ("entries", concat(map(f, sort(entries(path))))) + ]) + where f(fn) = attrs( + [ ("name", fn) + , ("file", dump(path + "/" + fn)) + ]) + + where: + + attrs(as) = concat(map(attr, as)) + encN(0) + attrs((a, b)) = encS(a) + encS(b) + + encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) + + encN(n) = 64-bit little-endian encoding of n. + + contents(path) = the contents of a regular file. + + sort(strings) = lexicographic sort by 8-bit value (strcmp). + + entries(path) = the entries of a directory, without `.' and + `..'. + + `+' denotes string concatenation. */ + +struct PathFilter +{ + virtual ~PathFilter() { } + virtual bool operator () (const Path & path) { return true; } +}; + +extern PathFilter defaultPathFilter; + +void dumpPath(const Path & path, Sink & sink, + PathFilter & filter = defaultPathFilter); + +struct ParseSink +{ + virtual void createDirectory(const Path & path) { }; + + virtual void createRegularFile(const Path & path) { }; + virtual void isExecutable() { }; + virtual void preallocateContents(unsigned long long size) { }; + virtual void receiveContents(unsigned char * data, unsigned int len) { }; + + virtual void createSymlink(const Path & path, const string & target) { }; +}; + +void parseDump(ParseSink & sink, Source & source); + +void restorePath(const Path & path, Source & source); + + +} diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc new file mode 100644 index 000000000000..de2c1ebd724f --- /dev/null +++ b/src/libutil/hash.cc @@ -0,0 +1,382 @@ +#include "config.h" + +#include <iostream> +#include <cstring> + +#ifdef HAVE_OPENSSL +#include <openssl/md5.h> +#include <openssl/sha.h> +#else +extern "C" { +#include "md5.h" +#include "sha1.h" +#include "sha256.h" +} +#endif + +#include "hash.hh" +#include "archive.hh" +#include "util.hh" + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + + +namespace nix { + + +Hash::Hash() +{ + type = htUnknown; + hashSize = 0; + memset(hash, 0, maxHashSize); +} + + +Hash::Hash(HashType type) +{ + this->type = type; + if (type == htMD5) hashSize = md5HashSize; + else if (type == htSHA1) hashSize = sha1HashSize; + else if (type == htSHA256) hashSize = sha256HashSize; + else throw Error("unknown hash type"); + assert(hashSize <= maxHashSize); + memset(hash, 0, maxHashSize); +} + + +bool Hash::operator == (const Hash & h2) const +{ + if (hashSize != h2.hashSize) return false; + for (unsigned int i = 0; i < hashSize; i++) + if (hash[i] != h2.hash[i]) return false; + return true; +} + + +bool Hash::operator != (const Hash & h2) const +{ + return !(*this == h2); +} + + +bool Hash::operator < (const Hash & h) const +{ + for (unsigned int i = 0; i < hashSize; i++) { + if (hash[i] < h.hash[i]) return true; + if (hash[i] > h.hash[i]) return false; + } + return false; +} + + +const string base16Chars = "0123456789abcdef"; + + +string printHash(const Hash & hash) +{ + char buf[hash.hashSize * 2]; + for (unsigned int i = 0; i < hash.hashSize; i++) { + buf[i * 2] = base16Chars[hash.hash[i] >> 4]; + buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f]; + } + return string(buf, hash.hashSize * 2); +} + + +Hash parseHash(HashType ht, const string & s) +{ + Hash hash(ht); + if (s.length() != hash.hashSize * 2) + throw Error(format("invalid hash `%1%'") % s); + for (unsigned int i = 0; i < hash.hashSize; i++) { + string s2(s, i * 2, 2); + if (!isxdigit(s2[0]) || !isxdigit(s2[1])) + throw Error(format("invalid hash `%1%'") % s); + std::istringstream str(s2); + int n; + str >> std::hex >> n; + hash.hash[i] = n; + } + return hash; +} + + +static unsigned char divMod(unsigned char * bytes, unsigned char y) +{ + unsigned int borrow = 0; + + int pos = Hash::maxHashSize - 1; + while (pos >= 0 && !bytes[pos]) --pos; + + for ( ; pos >= 0; --pos) { + unsigned int s = bytes[pos] + (borrow << 8); + unsigned int d = s / y; + borrow = s % y; + bytes[pos] = d; + } + + return borrow; +} + + +unsigned int hashLength32(const Hash & hash) +{ + return (hash.hashSize * 8 - 1) / 5 + 1; +} + + +// omitted: E O U T +const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; + + +string printHash32(const Hash & hash) +{ + Hash hash2(hash); + unsigned int len = hashLength32(hash); + + const char * chars = base32Chars.data(); + + string s(len, '0'); + + int pos = len - 1; + while (pos >= 0) { + unsigned char digit = divMod(hash2.hash, 32); + s[pos--] = chars[digit]; + } + + for (unsigned int i = 0; i < hash2.maxHashSize; ++i) + assert(hash2.hash[i] == 0); + + return s; +} + + +string printHash16or32(const Hash & hash) +{ + return hash.type == htMD5 ? printHash(hash) : printHash32(hash); +} + + +static bool mul(unsigned char * bytes, unsigned char y, int maxSize) +{ + unsigned char carry = 0; + + for (int pos = 0; pos < maxSize; ++pos) { + unsigned int m = bytes[pos] * y + carry; + bytes[pos] = m & 0xff; + carry = m >> 8; + } + + return carry; +} + + +static bool add(unsigned char * bytes, unsigned char y, int maxSize) +{ + unsigned char carry = y; + + for (int pos = 0; pos < maxSize; ++pos) { + unsigned int m = bytes[pos] + carry; + bytes[pos] = m & 0xff; + carry = m >> 8; + if (carry == 0) break; + } + + return carry; +} + + +Hash parseHash32(HashType ht, const string & s) +{ + Hash hash(ht); + + const char * chars = base32Chars.data(); + + for (unsigned int i = 0; i < s.length(); ++i) { + char c = s[i]; + unsigned char digit; + for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ + if (chars[digit] == c) break; + if (digit >= 32) + throw Error(format("invalid base-32 hash `%1%'") % s); + if (mul(hash.hash, 32, hash.hashSize) || + add(hash.hash, digit, hash.hashSize)) + throw Error(format("base-32 hash `%1%' is too large") % s); + } + + return hash; +} + + +Hash parseHash16or32(HashType ht, const string & s) +{ + Hash hash(ht); + if (s.size() == hash.hashSize * 2) + /* hexadecimal representation */ + hash = parseHash(ht, s); + else if (s.size() == hashLength32(hash)) + /* base-32 representation */ + hash = parseHash32(ht, s); + else + throw Error(format("hash `%1%' has wrong length for hash type `%2%'") + % s % printHashType(ht)); + return hash; +} + + +bool isHash(const string & s) +{ + if (s.length() != 32) return false; + for (int i = 0; i < 32; i++) { + char c = s[i]; + if (!((c >= '0' && c <= '9') || + (c >= 'a' && c <= 'f'))) + return false; + } + return true; +} + + +union Ctx +{ + MD5_CTX md5; + SHA_CTX sha1; + SHA256_CTX sha256; +}; + + +static void start(HashType ht, Ctx & ctx) +{ + if (ht == htMD5) MD5_Init(&ctx.md5); + else if (ht == htSHA1) SHA1_Init(&ctx.sha1); + else if (ht == htSHA256) SHA256_Init(&ctx.sha256); +} + + +static void update(HashType ht, Ctx & ctx, + const unsigned char * bytes, unsigned int len) +{ + if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); + else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); + else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len); +} + + +static void finish(HashType ht, Ctx & ctx, unsigned char * hash) +{ + if (ht == htMD5) MD5_Final(hash, &ctx.md5); + else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1); + else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256); +} + + +Hash hashString(HashType ht, const string & s) +{ + Ctx ctx; + Hash hash(ht); + start(ht, ctx); + update(ht, ctx, (const unsigned char *) s.data(), s.length()); + finish(ht, ctx, hash.hash); + return hash; +} + + +Hash hashFile(HashType ht, const Path & path) +{ + Ctx ctx; + Hash hash(ht); + start(ht, ctx); + + AutoCloseFD fd = open(path.c_str(), O_RDONLY); + if (fd == -1) throw SysError(format("opening file `%1%'") % path); + + unsigned char buf[8192]; + ssize_t n; + while ((n = read(fd, buf, sizeof(buf)))) { + checkInterrupt(); + if (n == -1) throw SysError(format("reading file `%1%'") % path); + update(ht, ctx, buf, n); + } + + finish(ht, ctx, hash.hash); + return hash; +} + + +HashSink::HashSink(HashType ht) : ht(ht) +{ + ctx = new Ctx; + bytes = 0; + start(ht, *ctx); +} + +HashSink::~HashSink() +{ + bufPos = 0; + delete ctx; +} + +void HashSink::write(const unsigned char * data, size_t len) +{ + bytes += len; + update(ht, *ctx, data, len); +} + +HashResult HashSink::finish() +{ + flush(); + Hash hash(ht); + nix::finish(ht, *ctx, hash.hash); + return HashResult(hash, bytes); +} + +HashResult HashSink::currentHash() +{ + flush(); + Ctx ctx2 = *ctx; + Hash hash(ht); + nix::finish(ht, ctx2, hash.hash); + return HashResult(hash, bytes); +} + + +HashResult hashPath( + HashType ht, const Path & path, PathFilter & filter) +{ + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish(); +} + + +Hash compressHash(const Hash & hash, unsigned int newSize) +{ + Hash h; + h.hashSize = newSize; + for (unsigned int i = 0; i < hash.hashSize; ++i) + h.hash[i % newSize] ^= hash.hash[i]; + return h; +} + + +HashType parseHashType(const string & s) +{ + if (s == "md5") return htMD5; + else if (s == "sha1") return htSHA1; + else if (s == "sha256") return htSHA256; + else return htUnknown; +} + + +string printHashType(HashType ht) +{ + if (ht == htMD5) return "md5"; + else if (ht == htSHA1) return "sha1"; + else if (ht == htSHA256) return "sha256"; + else throw Error("cannot print unknown hash type"); +} + + +} diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh new file mode 100644 index 000000000000..2c6f176ec74c --- /dev/null +++ b/src/libutil/hash.hh @@ -0,0 +1,113 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" + + +namespace nix { + + +typedef enum { htUnknown, htMD5, htSHA1, htSHA256 } HashType; + + +const int md5HashSize = 16; +const int sha1HashSize = 20; +const int sha256HashSize = 32; + +extern const string base32Chars; + + +struct Hash +{ + static const unsigned int maxHashSize = 32; + unsigned int hashSize; + unsigned char hash[maxHashSize]; + + HashType type; + + /* Create an unusable hash object. */ + Hash(); + + /* Create a zero-filled hash object. */ + Hash(HashType type); + + /* Check whether two hash are equal. */ + bool operator == (const Hash & h2) const; + + /* Check whether two hash are not equal. */ + bool operator != (const Hash & h2) const; + + /* For sorting. */ + bool operator < (const Hash & h) const; +}; + + +/* Convert a hash to a hexadecimal representation. */ +string printHash(const Hash & hash); + +/* Parse a hexadecimal representation of a hash code. */ +Hash parseHash(HashType ht, const string & s); + +/* Returns the length of a base-32 hash representation. */ +unsigned int hashLength32(const Hash & hash); + +/* Convert a hash to a base-32 representation. */ +string printHash32(const Hash & hash); + +/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ +string printHash16or32(const Hash & hash); + +/* Parse a base-32 representation of a hash code. */ +Hash parseHash32(HashType ht, const string & s); + +/* Parse a base-16 or base-32 representation of a hash code. */ +Hash parseHash16or32(HashType ht, const string & s); + +/* Verify that the given string is a valid hash code. */ +bool isHash(const string & s); + +/* Compute the hash of the given string. */ +Hash hashString(HashType ht, const string & s); + +/* Compute the hash of the given file. */ +Hash hashFile(HashType ht, const Path & path); + +/* Compute the hash of the given path. The hash is defined as + (essentially) hashString(ht, dumpPath(path)). */ +struct PathFilter; +extern PathFilter defaultPathFilter; +typedef std::pair<Hash, unsigned long long> HashResult; +HashResult hashPath(HashType ht, const Path & path, + PathFilter & filter = defaultPathFilter); + +/* Compress a hash to the specified number of bytes by cyclically + XORing bytes together. */ +Hash compressHash(const Hash & hash, unsigned int newSize); + +/* Parse a string representing a hash type. */ +HashType parseHashType(const string & s); + +/* And the reverse. */ +string printHashType(HashType ht); + + +union Ctx; + +class HashSink : public BufferedSink +{ +private: + HashType ht; + Ctx * ctx; + unsigned long long bytes; + +public: + HashSink(HashType ht); + HashSink(const HashSink & h); + ~HashSink(); + void write(const unsigned char * data, size_t len); + HashResult finish(); + HashResult currentHash(); +}; + + +} diff --git a/src/libutil/local.mk b/src/libutil/local.mk new file mode 100644 index 000000000000..8af2e78d9ce4 --- /dev/null +++ b/src/libutil/local.mk @@ -0,0 +1,15 @@ +libraries += libutil + +libutil_NAME = libnixutil + +libutil_DIR := $(d) + +libutil_SOURCES := $(wildcard $(d)/*.cc) + +ifeq ($(HAVE_OPENSSL), 1) + libutil_LDFLAGS = $(OPENSSL_LIBS) +else + libutil_SOURCES += $(d)/md5.c $(d)/sha1.c $(d)/sha256.c +endif + +libutil_LIBS = libformat diff --git a/src/libutil/md32_common.h b/src/libutil/md32_common.h new file mode 100644 index 000000000000..0cbcfaf8a20b --- /dev/null +++ b/src/libutil/md32_common.h @@ -0,0 +1,620 @@ +/* crypto/md32_common.h */ +/* ==================================================================== + * Copyright (c) 1999-2002 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * licensing@OpenSSL.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + +/* + * This is a generic 32 bit "collector" for message digest algorithms. + * Whenever needed it collects input character stream into chunks of + * 32 bit values and invokes a block function that performs actual hash + * calculations. + * + * Porting guide. + * + * Obligatory macros: + * + * DATA_ORDER_IS_BIG_ENDIAN or DATA_ORDER_IS_LITTLE_ENDIAN + * this macro defines byte order of input stream. + * HASH_CBLOCK + * size of a unit chunk HASH_BLOCK operates on. + * HASH_LONG + * has to be at lest 32 bit wide, if it's wider, then + * HASH_LONG_LOG2 *has to* be defined along + * HASH_CTX + * context structure that at least contains following + * members: + * typedef struct { + * ... + * HASH_LONG Nl,Nh; + * HASH_LONG data[HASH_LBLOCK]; + * unsigned int num; + * ... + * } HASH_CTX; + * HASH_UPDATE + * name of "Update" function, implemented here. + * HASH_TRANSFORM + * name of "Transform" function, implemented here. + * HASH_FINAL + * name of "Final" function, implemented here. + * HASH_BLOCK_HOST_ORDER + * name of "block" function treating *aligned* input message + * in host byte order, implemented externally. + * HASH_BLOCK_DATA_ORDER + * name of "block" function treating *unaligned* input message + * in original (data) byte order, implemented externally (it + * actually is optional if data and host are of the same + * "endianess"). + * HASH_MAKE_STRING + * macro convering context variables to an ASCII hash string. + * + * Optional macros: + * + * B_ENDIAN or L_ENDIAN + * defines host byte-order. + * HASH_LONG_LOG2 + * defaults to 2 if not states otherwise. + * HASH_LBLOCK + * assumed to be HASH_CBLOCK/4 if not stated otherwise. + * HASH_BLOCK_DATA_ORDER_ALIGNED + * alternative "block" function capable of treating + * aligned input message in original (data) order, + * implemented externally. + * + * MD5 example: + * + * #define DATA_ORDER_IS_LITTLE_ENDIAN + * + * #define HASH_LONG MD5_LONG + * #define HASH_LONG_LOG2 MD5_LONG_LOG2 + * #define HASH_CTX MD5_CTX + * #define HASH_CBLOCK MD5_CBLOCK + * #define HASH_LBLOCK MD5_LBLOCK + * #define HASH_UPDATE MD5_Update + * #define HASH_TRANSFORM MD5_Transform + * #define HASH_FINAL MD5_Final + * #define HASH_BLOCK_HOST_ORDER md5_block_host_order + * #define HASH_BLOCK_DATA_ORDER md5_block_data_order + * + * <appro@fy.chalmers.se> + */ + +#if !defined(DATA_ORDER_IS_BIG_ENDIAN) && !defined(DATA_ORDER_IS_LITTLE_ENDIAN) +#error "DATA_ORDER must be defined!" +#endif + +#ifndef HASH_CBLOCK +#error "HASH_CBLOCK must be defined!" +#endif +#ifndef HASH_LONG +#error "HASH_LONG must be defined!" +#endif +#ifndef HASH_CTX +#error "HASH_CTX must be defined!" +#endif + +#ifndef HASH_UPDATE +#error "HASH_UPDATE must be defined!" +#endif +#ifndef HASH_TRANSFORM +#error "HASH_TRANSFORM must be defined!" +#endif +#ifndef HASH_FINAL +#error "HASH_FINAL must be defined!" +#endif + +#ifndef HASH_BLOCK_HOST_ORDER +#error "HASH_BLOCK_HOST_ORDER must be defined!" +#endif + +#if 0 +/* + * Moved below as it's required only if HASH_BLOCK_DATA_ORDER_ALIGNED + * isn't defined. + */ +#ifndef HASH_BLOCK_DATA_ORDER +#error "HASH_BLOCK_DATA_ORDER must be defined!" +#endif +#endif + +#ifndef HASH_LBLOCK +#define HASH_LBLOCK (HASH_CBLOCK/4) +#endif + +#ifndef HASH_LONG_LOG2 +#define HASH_LONG_LOG2 2 +#endif + +/* + * Engage compiler specific rotate intrinsic function if available. + */ +#undef ROTATE +#ifndef PEDANTIC +# if defined(_MSC_VER) || defined(__ICC) +# define ROTATE(a,n) _lrotl(a,n) +# elif defined(__MWERKS__) +# if defined(__POWERPC__) +# define ROTATE(a,n) __rlwinm(a,n,0,31) +# elif defined(__MC68K__) + /* Motorola specific tweak. <appro@fy.chalmers.se> */ +# define ROTATE(a,n) ( n<24 ? __rol(a,n) : __ror(a,32-n) ) +# else +# define ROTATE(a,n) __rol(a,n) +# endif +# elif defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) + /* + * Some GNU C inline assembler templates. Note that these are + * rotates by *constant* number of bits! But that's exactly + * what we need here... + * <appro@fy.chalmers.se> + */ +# if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) +# define ROTATE(a,n) ({ register unsigned int ret; \ + asm ( \ + "roll %1,%0" \ + : "=r"(ret) \ + : "I"(n), "0"(a) \ + : "cc"); \ + ret; \ + }) +# elif defined(__powerpc) || defined(__ppc__) || defined(__powerpc64__) +# define ROTATE(a,n) ({ register unsigned int ret; \ + asm ( \ + "rlwinm %0,%1,%2,0,31" \ + : "=r"(ret) \ + : "r"(a), "I"(n)); \ + ret; \ + }) +# endif +# endif +#endif /* PEDANTIC */ + +#if HASH_LONG_LOG2==2 /* Engage only if sizeof(HASH_LONG)== 4 */ +/* A nice byte order reversal from Wei Dai <weidai@eskimo.com> */ +#ifdef ROTATE +/* 5 instructions with rotate instruction, else 9 */ +#define REVERSE_FETCH32(a,l) ( \ + l=*(const HASH_LONG *)(a), \ + ((ROTATE(l,8)&0x00FF00FF)|(ROTATE((l&0x00FF00FF),24))) \ + ) +#else +/* 6 instructions with rotate instruction, else 8 */ +#define REVERSE_FETCH32(a,l) ( \ + l=*(const HASH_LONG *)(a), \ + l=(((l>>8)&0x00FF00FF)|((l&0x00FF00FF)<<8)), \ + ROTATE(l,16) \ + ) +/* + * Originally the middle line started with l=(((l&0xFF00FF00)>>8)|... + * It's rewritten as above for two reasons: + * - RISCs aren't good at long constants and have to explicitely + * compose 'em with several (well, usually 2) instructions in a + * register before performing the actual operation and (as you + * already realized:-) having same constant should inspire the + * compiler to permanently allocate the only register for it; + * - most modern CPUs have two ALUs, but usually only one has + * circuitry for shifts:-( this minor tweak inspires compiler + * to schedule shift instructions in a better way... + * + * <appro@fy.chalmers.se> + */ +#endif +#endif + +#ifndef ROTATE +#define ROTATE(a,n) (((a)<<(n))|(((a)&0xffffffff)>>(32-(n)))) +#endif + +/* + * Make some obvious choices. E.g., HASH_BLOCK_DATA_ORDER_ALIGNED + * and HASH_BLOCK_HOST_ORDER ought to be the same if input data + * and host are of the same "endianess". It's possible to mask + * this with blank #define HASH_BLOCK_DATA_ORDER though... + * + * <appro@fy.chalmers.se> + */ +#if defined(B_ENDIAN) +# if defined(DATA_ORDER_IS_BIG_ENDIAN) +# if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) && HASH_LONG_LOG2==2 +# define HASH_BLOCK_DATA_ORDER_ALIGNED HASH_BLOCK_HOST_ORDER +# endif +# endif +#elif defined(L_ENDIAN) +# if defined(DATA_ORDER_IS_LITTLE_ENDIAN) +# if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) && HASH_LONG_LOG2==2 +# define HASH_BLOCK_DATA_ORDER_ALIGNED HASH_BLOCK_HOST_ORDER +# endif +# endif +#endif + +#if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) +#ifndef HASH_BLOCK_DATA_ORDER +#error "HASH_BLOCK_DATA_ORDER must be defined!" +#endif +#endif + +#if defined(DATA_ORDER_IS_BIG_ENDIAN) + +#ifndef PEDANTIC +# if defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) +# if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) + /* + * This gives ~30-40% performance improvement in SHA-256 compiled + * with gcc [on P4]. Well, first macro to be frank. We can pull + * this trick on x86* platforms only, because these CPUs can fetch + * unaligned data without raising an exception. + */ +# define HOST_c2l(c,l) ({ unsigned int r=*((const unsigned int *)(c)); \ + asm ("bswapl %0":"=r"(r):"0"(r)); \ + (c)+=4; (l)=r; }) +# define HOST_l2c(l,c) ({ unsigned int r=(l); \ + asm ("bswapl %0":"=r"(r):"0"(r)); \ + *((unsigned int *)(c))=r; (c)+=4; r; }) +# endif +# endif +#endif + +#ifndef HOST_c2l +#define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++)))<<24), \ + l|=(((unsigned long)(*((c)++)))<<16), \ + l|=(((unsigned long)(*((c)++)))<< 8), \ + l|=(((unsigned long)(*((c)++))) ), \ + l) +#endif +#define HOST_p_c2l(c,l,n) { \ + switch (n) { \ + case 0: l =((unsigned long)(*((c)++)))<<24; \ + case 1: l|=((unsigned long)(*((c)++)))<<16; \ + case 2: l|=((unsigned long)(*((c)++)))<< 8; \ + case 3: l|=((unsigned long)(*((c)++))); \ + } } +#define HOST_p_c2l_p(c,l,sc,len) { \ + switch (sc) { \ + case 0: l =((unsigned long)(*((c)++)))<<24; \ + if (--len == 0) break; \ + case 1: l|=((unsigned long)(*((c)++)))<<16; \ + if (--len == 0) break; \ + case 2: l|=((unsigned long)(*((c)++)))<< 8; \ + } } +/* NOTE the pointer is not incremented at the end of this */ +#define HOST_c2l_p(c,l,n) { \ + l=0; (c)+=n; \ + switch (n) { \ + case 3: l =((unsigned long)(*(--(c))))<< 8; \ + case 2: l|=((unsigned long)(*(--(c))))<<16; \ + case 1: l|=((unsigned long)(*(--(c))))<<24; \ + } } +#ifndef HOST_l2c +#define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l)>>24)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16)&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ + *((c)++)=(unsigned char)(((l) )&0xff), \ + l) +#endif + +#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) + +#if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) + /* See comment in DATA_ORDER_IS_BIG_ENDIAN section. */ +# define HOST_c2l(c,l) ((l)=*((const unsigned int *)(c)), (c)+=4, l) +# define HOST_l2c(l,c) (*((unsigned int *)(c))=(l), (c)+=4, l) +#endif + +#ifndef HOST_c2l +#define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++))) ), \ + l|=(((unsigned long)(*((c)++)))<< 8), \ + l|=(((unsigned long)(*((c)++)))<<16), \ + l|=(((unsigned long)(*((c)++)))<<24), \ + l) +#endif +#define HOST_p_c2l(c,l,n) { \ + switch (n) { \ + case 0: l =((unsigned long)(*((c)++))); \ + case 1: l|=((unsigned long)(*((c)++)))<< 8; \ + case 2: l|=((unsigned long)(*((c)++)))<<16; \ + case 3: l|=((unsigned long)(*((c)++)))<<24; \ + } } +#define HOST_p_c2l_p(c,l,sc,len) { \ + switch (sc) { \ + case 0: l =((unsigned long)(*((c)++))); \ + if (--len == 0) break; \ + case 1: l|=((unsigned long)(*((c)++)))<< 8; \ + if (--len == 0) break; \ + case 2: l|=((unsigned long)(*((c)++)))<<16; \ + } } +/* NOTE the pointer is not incremented at the end of this */ +#define HOST_c2l_p(c,l,n) { \ + l=0; (c)+=n; \ + switch (n) { \ + case 3: l =((unsigned long)(*(--(c))))<<16; \ + case 2: l|=((unsigned long)(*(--(c))))<< 8; \ + case 1: l|=((unsigned long)(*(--(c)))); \ + } } +#ifndef HOST_l2c +#define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16)&0xff), \ + *((c)++)=(unsigned char)(((l)>>24)&0xff), \ + l) +#endif + +#endif + +/* + * Time for some action:-) + */ + +int HASH_UPDATE (HASH_CTX *c, const void *data_, size_t len) + { + const unsigned char *data=data_; + register HASH_LONG * p; + register HASH_LONG l; + size_t sw,sc,ew,ec; + + if (len==0) return 1; + + l=(c->Nl+(((HASH_LONG)len)<<3))&0xffffffffUL; + /* 95-05-24 eay Fixed a bug with the overflow handling, thanks to + * Wei Dai <weidai@eskimo.com> for pointing it out. */ + if (l < c->Nl) /* overflow */ + c->Nh++; + c->Nh+=(len>>29); /* might cause compiler warning on 16-bit */ + c->Nl=l; + + if (c->num != 0) + { + p=c->data; + sw=c->num>>2; + sc=c->num&0x03; + + if ((c->num+len) >= HASH_CBLOCK) + { + l=p[sw]; HOST_p_c2l(data,l,sc); p[sw++]=l; + for (; sw<HASH_LBLOCK; sw++) + { + HOST_c2l(data,l); p[sw]=l; + } + HASH_BLOCK_HOST_ORDER (c,p,1); + len-=(HASH_CBLOCK-c->num); + c->num=0; + /* drop through and do the rest */ + } + else + { + c->num+=(unsigned int)len; + if ((sc+len) < 4) /* ugly, add char's to a word */ + { + l=p[sw]; HOST_p_c2l_p(data,l,sc,len); p[sw]=l; + } + else + { + ew=(c->num>>2); + ec=(c->num&0x03); + if (sc) + l=p[sw]; + HOST_p_c2l(data,l,sc); + p[sw++]=l; + for (; sw < ew; sw++) + { + HOST_c2l(data,l); p[sw]=l; + } + if (ec) + { + HOST_c2l_p(data,l,ec); p[sw]=l; + } + } + return 1; + } + } + + sw=len/HASH_CBLOCK; + if (sw > 0) + { +#if defined(HASH_BLOCK_DATA_ORDER_ALIGNED) + /* + * Note that HASH_BLOCK_DATA_ORDER_ALIGNED gets defined + * only if sizeof(HASH_LONG)==4. + */ + if ((((size_t)data)%4) == 0) + { + /* data is properly aligned so that we can cast it: */ + HASH_BLOCK_DATA_ORDER_ALIGNED (c,(const HASH_LONG *)data,sw); + sw*=HASH_CBLOCK; + data+=sw; + len-=sw; + } + else +#if !defined(HASH_BLOCK_DATA_ORDER) + while (sw--) + { + memcpy (p=c->data,data,HASH_CBLOCK); + HASH_BLOCK_DATA_ORDER_ALIGNED(c,p,1); + data+=HASH_CBLOCK; + len-=HASH_CBLOCK; + } +#endif +#endif +#if defined(HASH_BLOCK_DATA_ORDER) + { + HASH_BLOCK_DATA_ORDER(c,data,sw); + sw*=HASH_CBLOCK; + data+=sw; + len-=sw; + } +#endif + } + + if (len!=0) + { + p = c->data; + c->num = len; + ew=len>>2; /* words to copy */ + ec=len&0x03; + for (; ew; ew--,p++) + { + HOST_c2l(data,l); *p=l; + } + HOST_c2l_p(data,l,ec); + *p=l; + } + return 1; + } + + +void HASH_TRANSFORM (HASH_CTX *c, const unsigned char *data) + { +#if defined(HASH_BLOCK_DATA_ORDER_ALIGNED) + if ((((size_t)data)%4) == 0) + /* data is properly aligned so that we can cast it: */ + HASH_BLOCK_DATA_ORDER_ALIGNED (c,(const HASH_LONG *)data,1); + else +#if !defined(HASH_BLOCK_DATA_ORDER) + { + memcpy (c->data,data,HASH_CBLOCK); + HASH_BLOCK_DATA_ORDER_ALIGNED (c,c->data,1); + } +#endif +#endif +#if defined(HASH_BLOCK_DATA_ORDER) + HASH_BLOCK_DATA_ORDER (c,data,1); +#endif + } + + +int HASH_FINAL (unsigned char *md, HASH_CTX *c) + { + register HASH_LONG *p; + register unsigned long l; + register int i,j; + static const unsigned char end[4]={0x80,0x00,0x00,0x00}; + const unsigned char *cp=end; + + /* c->num should definitly have room for at least one more byte. */ + p=c->data; + i=c->num>>2; + j=c->num&0x03; + +#if 0 + /* purify often complains about the following line as an + * Uninitialized Memory Read. While this can be true, the + * following p_c2l macro will reset l when that case is true. + * This is because j&0x03 contains the number of 'valid' bytes + * already in p[i]. If and only if j&0x03 == 0, the UMR will + * occur but this is also the only time p_c2l will do + * l= *(cp++) instead of l|= *(cp++) + * Many thanks to Alex Tang <altitude@cic.net> for pickup this + * 'potential bug' */ +#ifdef PURIFY + if (j==0) p[i]=0; /* Yeah, but that's not the way to fix it:-) */ +#endif + l=p[i]; +#else + l = (j==0) ? 0 : p[i]; +#endif + HOST_p_c2l(cp,l,j); p[i++]=l; /* i is the next 'undefined word' */ + + if (i>(HASH_LBLOCK-2)) /* save room for Nl and Nh */ + { + if (i<HASH_LBLOCK) p[i]=0; + HASH_BLOCK_HOST_ORDER (c,p,1); + i=0; + } + for (; i<(HASH_LBLOCK-2); i++) + p[i]=0; + +#if defined(DATA_ORDER_IS_BIG_ENDIAN) + p[HASH_LBLOCK-2]=c->Nh; + p[HASH_LBLOCK-1]=c->Nl; +#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) + p[HASH_LBLOCK-2]=c->Nl; + p[HASH_LBLOCK-1]=c->Nh; +#endif + HASH_BLOCK_HOST_ORDER (c,p,1); + +#ifndef HASH_MAKE_STRING +#error "HASH_MAKE_STRING must be defined!" +#else + HASH_MAKE_STRING(c,md); +#endif + + c->num=0; + /* clear stuff, HASH_BLOCK may be leaving some stuff on the stack + * but I'm not worried :-) + OPENSSL_cleanse((void *)c,sizeof(HASH_CTX)); + */ + return 1; + } + +#ifndef MD32_REG_T +#define MD32_REG_T long +/* + * This comment was originaly written for MD5, which is why it + * discusses A-D. But it basically applies to all 32-bit digests, + * which is why it was moved to common header file. + * + * In case you wonder why A-D are declared as long and not + * as MD5_LONG. Doing so results in slight performance + * boost on LP64 architectures. The catch is we don't + * really care if 32 MSBs of a 64-bit register get polluted + * with eventual overflows as we *save* only 32 LSBs in + * *either* case. Now declaring 'em long excuses the compiler + * from keeping 32 MSBs zeroed resulting in 13% performance + * improvement under SPARC Solaris7/64 and 5% under AlphaLinux. + * Well, to be honest it should say that this *prevents* + * performance degradation. + * <appro@fy.chalmers.se> + * Apparently there're LP64 compilers that generate better + * code if A-D are declared int. Most notably GCC-x86_64 + * generates better code. + * <appro@fy.chalmers.se> + */ +#endif diff --git a/src/libutil/md5.c b/src/libutil/md5.c new file mode 100644 index 000000000000..b31640cdcced --- /dev/null +++ b/src/libutil/md5.c @@ -0,0 +1,365 @@ +/* Functions to compute MD5 message digest of files or memory blocks. + according to the definition of MD5 in RFC 1321 from April 1992. + Copyright (C) 1995,1996,1997,1999,2000,2001 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +/* Written by Ulrich Drepper <drepper@gnu.ai.mit.edu>, 1995. */ + +#include <sys/types.h> + +#include <stdlib.h> +#include <string.h> + +#include "md5.h" + + +static md5_uint32 SWAP(md5_uint32 n) +{ + static int checked = 0; + static int bigendian = 0; + static md5_uint32 test; + + if (!checked) { + test = 1; + if (* (char *) &test == 0) + bigendian = 1; + checked = 1; + } + + if (bigendian) + return (((n) << 24) | (((n) & 0xff00) << 8) | (((n) >> 8) & 0xff00) | ((n) >> 24)); + else + return n; +} + + +/* This array contains the bytes used to pad the buffer to the next + 64-byte boundary. (RFC 1321, 3.1: Step 1) */ +static const unsigned char fillbuf[64] = { 0x80, 0 /* , 0, 0, ... */ }; + + +/* Initialize structure containing state of computation. + (RFC 1321, 3.3: Step 3) */ +void +MD5_Init (ctx) + struct MD5_CTX *ctx; +{ + ctx->A = 0x67452301; + ctx->B = 0xefcdab89; + ctx->C = 0x98badcfe; + ctx->D = 0x10325476; + + ctx->total[0] = ctx->total[1] = 0; + ctx->buflen = 0; +} + +/* Put result from CTX in first 16 bytes following RESBUF. The result + must be in little endian byte order. + + IMPORTANT: On some systems it is required that RESBUF is correctly + aligned for a 32 bits value. */ +void * +md5_read_ctx (ctx, resbuf) + const struct MD5_CTX *ctx; + void *resbuf; +{ + ((md5_uint32 *) resbuf)[0] = SWAP (ctx->A); + ((md5_uint32 *) resbuf)[1] = SWAP (ctx->B); + ((md5_uint32 *) resbuf)[2] = SWAP (ctx->C); + ((md5_uint32 *) resbuf)[3] = SWAP (ctx->D); + + return resbuf; +} + +/* Process the remaining bytes in the internal buffer and the usual + prolog according to the standard and write the result to RESBUF. + + IMPORTANT: On some systems it is required that RESBUF is correctly + aligned for a 32 bits value. */ +void * +MD5_Final (resbuf, ctx) + void *resbuf; + struct MD5_CTX *ctx; +{ + /* Take yet unprocessed bytes into account. */ + md5_uint32 bytes = ctx->buflen; + size_t pad; + + /* Now count remaining bytes. */ + ctx->total[0] += bytes; + if (ctx->total[0] < bytes) + ++ctx->total[1]; + + pad = bytes >= 56 ? 64 + 56 - bytes : 56 - bytes; + memcpy (&ctx->buffer[bytes], fillbuf, pad); + + /* Put the 64-bit file length in *bits* at the end of the buffer. */ + *(md5_uint32 *) &ctx->buffer[bytes + pad] = SWAP (ctx->total[0] << 3); + *(md5_uint32 *) &ctx->buffer[bytes + pad + 4] = SWAP ((ctx->total[1] << 3) | + (ctx->total[0] >> 29)); + + /* Process last bytes. */ + md5_process_block (ctx->buffer, bytes + pad + 8, ctx); + + return md5_read_ctx (ctx, resbuf); +} + +void +MD5_Update (ctx, buffer, len) + struct MD5_CTX *ctx; + const void *buffer; + size_t len; +{ + /* When we already have some bits in our internal buffer concatenate + both inputs first. */ + if (ctx->buflen != 0) + { + size_t left_over = ctx->buflen; + size_t add = 128 - left_over > len ? len : 128 - left_over; + + memcpy (&ctx->buffer[left_over], buffer, add); + ctx->buflen += add; + + if (ctx->buflen > 64) + { + md5_process_block (ctx->buffer, ctx->buflen & ~63, ctx); + + ctx->buflen &= 63; + /* The regions in the following copy operation cannot overlap. */ + memcpy (ctx->buffer, &ctx->buffer[(left_over + add) & ~63], + ctx->buflen); + } + + buffer = (const char *) buffer + add; + len -= add; + } + + /* Process available complete blocks. */ + if (len >= 64) + { +#if !_STRING_ARCH_unaligned +/* To check alignment gcc has an appropriate operator. Other + compilers don't. */ +# if __GNUC__ >= 2 +# define UNALIGNED_P(p) (((md5_uintptr) p) % __alignof__ (md5_uint32) != 0) +# else +# define UNALIGNED_P(p) (((md5_uintptr) p) % sizeof (md5_uint32) != 0) +# endif + if (UNALIGNED_P (buffer)) + while (len > 64) + { + md5_process_block (memcpy (ctx->buffer, buffer, 64), 64, ctx); + buffer = (const char *) buffer + 64; + len -= 64; + } + else +#endif + { + md5_process_block (buffer, len & ~63, ctx); + buffer = (const char *) buffer + (len & ~63); + len &= 63; + } + } + + /* Move remaining bytes in internal buffer. */ + if (len > 0) + { + size_t left_over = ctx->buflen; + + memcpy (&ctx->buffer[left_over], buffer, len); + left_over += len; + if (left_over >= 64) + { + md5_process_block (ctx->buffer, 64, ctx); + left_over -= 64; + memcpy (ctx->buffer, &ctx->buffer[64], left_over); + } + ctx->buflen = left_over; + } +} + + +/* These are the four functions used in the four steps of the MD5 algorithm + and defined in the RFC 1321. The first function is a little bit optimized + (as found in Colin Plumbs public domain implementation). */ +/* #define FF(b, c, d) ((b & c) | (~b & d)) */ +#define FF(b, c, d) (d ^ (b & (c ^ d))) +#define FG(b, c, d) FF (d, b, c) +#define FH(b, c, d) (b ^ c ^ d) +#define FI(b, c, d) (c ^ (b | ~d)) + +/* Process LEN bytes of BUFFER, accumulating context into CTX. + It is assumed that LEN % 64 == 0. */ + +void +md5_process_block (buffer, len, ctx) + const void *buffer; + size_t len; + struct MD5_CTX *ctx; +{ + md5_uint32 correct_words[16]; + const md5_uint32 *words = buffer; + size_t nwords = len / sizeof (md5_uint32); + const md5_uint32 *endp = words + nwords; + md5_uint32 A = ctx->A; + md5_uint32 B = ctx->B; + md5_uint32 C = ctx->C; + md5_uint32 D = ctx->D; + + /* First increment the byte count. RFC 1321 specifies the possible + length of the file up to 2^64 bits. Here we only compute the + number of bytes. Do a double word increment. */ + ctx->total[0] += len; + if (ctx->total[0] < len) + ++ctx->total[1]; + + /* Process all bytes in the buffer with 64 bytes in each round of + the loop. */ + while (words < endp) + { + md5_uint32 *cwp = correct_words; + md5_uint32 A_save = A; + md5_uint32 B_save = B; + md5_uint32 C_save = C; + md5_uint32 D_save = D; + + /* First round: using the given function, the context and a constant + the next context is computed. Because the algorithms processing + unit is a 32-bit word and it is determined to work on words in + little endian byte order we perhaps have to change the byte order + before the computation. To reduce the work for the next steps + we store the swapped words in the array CORRECT_WORDS. */ + +#define OP(a, b, c, d, s, T) \ + do \ + { \ + a += FF (b, c, d) + (*cwp++ = SWAP (*words)) + T; \ + ++words; \ + CYCLIC (a, s); \ + a += b; \ + } \ + while (0) + + /* It is unfortunate that C does not provide an operator for + cyclic rotation. Hope the C compiler is smart enough. */ +#define CYCLIC(w, s) (w = (w << s) | (w >> (32 - s))) + + /* Before we start, one word to the strange constants. + They are defined in RFC 1321 as + + T[i] = (int) (4294967296.0 * fabs (sin (i))), i=1..64 + */ + + /* Round 1. */ + OP (A, B, C, D, 7, 0xd76aa478); + OP (D, A, B, C, 12, 0xe8c7b756); + OP (C, D, A, B, 17, 0x242070db); + OP (B, C, D, A, 22, 0xc1bdceee); + OP (A, B, C, D, 7, 0xf57c0faf); + OP (D, A, B, C, 12, 0x4787c62a); + OP (C, D, A, B, 17, 0xa8304613); + OP (B, C, D, A, 22, 0xfd469501); + OP (A, B, C, D, 7, 0x698098d8); + OP (D, A, B, C, 12, 0x8b44f7af); + OP (C, D, A, B, 17, 0xffff5bb1); + OP (B, C, D, A, 22, 0x895cd7be); + OP (A, B, C, D, 7, 0x6b901122); + OP (D, A, B, C, 12, 0xfd987193); + OP (C, D, A, B, 17, 0xa679438e); + OP (B, C, D, A, 22, 0x49b40821); + + /* For the second to fourth round we have the possibly swapped words + in CORRECT_WORDS. Redefine the macro to take an additional first + argument specifying the function to use. */ +#undef OP +#define OP(f, a, b, c, d, k, s, T) \ + do \ + { \ + a += f (b, c, d) + correct_words[k] + T; \ + CYCLIC (a, s); \ + a += b; \ + } \ + while (0) + + /* Round 2. */ + OP (FG, A, B, C, D, 1, 5, 0xf61e2562); + OP (FG, D, A, B, C, 6, 9, 0xc040b340); + OP (FG, C, D, A, B, 11, 14, 0x265e5a51); + OP (FG, B, C, D, A, 0, 20, 0xe9b6c7aa); + OP (FG, A, B, C, D, 5, 5, 0xd62f105d); + OP (FG, D, A, B, C, 10, 9, 0x02441453); + OP (FG, C, D, A, B, 15, 14, 0xd8a1e681); + OP (FG, B, C, D, A, 4, 20, 0xe7d3fbc8); + OP (FG, A, B, C, D, 9, 5, 0x21e1cde6); + OP (FG, D, A, B, C, 14, 9, 0xc33707d6); + OP (FG, C, D, A, B, 3, 14, 0xf4d50d87); + OP (FG, B, C, D, A, 8, 20, 0x455a14ed); + OP (FG, A, B, C, D, 13, 5, 0xa9e3e905); + OP (FG, D, A, B, C, 2, 9, 0xfcefa3f8); + OP (FG, C, D, A, B, 7, 14, 0x676f02d9); + OP (FG, B, C, D, A, 12, 20, 0x8d2a4c8a); + + /* Round 3. */ + OP (FH, A, B, C, D, 5, 4, 0xfffa3942); + OP (FH, D, A, B, C, 8, 11, 0x8771f681); + OP (FH, C, D, A, B, 11, 16, 0x6d9d6122); + OP (FH, B, C, D, A, 14, 23, 0xfde5380c); + OP (FH, A, B, C, D, 1, 4, 0xa4beea44); + OP (FH, D, A, B, C, 4, 11, 0x4bdecfa9); + OP (FH, C, D, A, B, 7, 16, 0xf6bb4b60); + OP (FH, B, C, D, A, 10, 23, 0xbebfbc70); + OP (FH, A, B, C, D, 13, 4, 0x289b7ec6); + OP (FH, D, A, B, C, 0, 11, 0xeaa127fa); + OP (FH, C, D, A, B, 3, 16, 0xd4ef3085); + OP (FH, B, C, D, A, 6, 23, 0x04881d05); + OP (FH, A, B, C, D, 9, 4, 0xd9d4d039); + OP (FH, D, A, B, C, 12, 11, 0xe6db99e5); + OP (FH, C, D, A, B, 15, 16, 0x1fa27cf8); + OP (FH, B, C, D, A, 2, 23, 0xc4ac5665); + + /* Round 4. */ + OP (FI, A, B, C, D, 0, 6, 0xf4292244); + OP (FI, D, A, B, C, 7, 10, 0x432aff97); + OP (FI, C, D, A, B, 14, 15, 0xab9423a7); + OP (FI, B, C, D, A, 5, 21, 0xfc93a039); + OP (FI, A, B, C, D, 12, 6, 0x655b59c3); + OP (FI, D, A, B, C, 3, 10, 0x8f0ccc92); + OP (FI, C, D, A, B, 10, 15, 0xffeff47d); + OP (FI, B, C, D, A, 1, 21, 0x85845dd1); + OP (FI, A, B, C, D, 8, 6, 0x6fa87e4f); + OP (FI, D, A, B, C, 15, 10, 0xfe2ce6e0); + OP (FI, C, D, A, B, 6, 15, 0xa3014314); + OP (FI, B, C, D, A, 13, 21, 0x4e0811a1); + OP (FI, A, B, C, D, 4, 6, 0xf7537e82); + OP (FI, D, A, B, C, 11, 10, 0xbd3af235); + OP (FI, C, D, A, B, 2, 15, 0x2ad7d2bb); + OP (FI, B, C, D, A, 9, 21, 0xeb86d391); + + /* Add the starting values of the context. */ + A += A_save; + B += B_save; + C += C_save; + D += D_save; + } + + /* Put checksum in context given as argument. */ + ctx->A = A; + ctx->B = B; + ctx->C = C; + ctx->D = D; +} diff --git a/src/libutil/md5.h b/src/libutil/md5.h new file mode 100644 index 000000000000..228d4972320f --- /dev/null +++ b/src/libutil/md5.h @@ -0,0 +1,82 @@ +/* Declaration of functions and data types used for MD5 sum computing + library functions. + Copyright (C) 1995,1996,1997,1999,2000,2001 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#ifndef _MD5_H +#define _MD5_H 1 + +#include <inttypes.h> +typedef uint32_t md5_uint32; +typedef uintptr_t md5_uintptr; + +/* Structure to save state of computation between the single steps. */ +struct MD5_CTX +{ + md5_uint32 A; + md5_uint32 B; + md5_uint32 C; + md5_uint32 D; + + md5_uint32 total[2]; + md5_uint32 buflen; + char buffer[128] __attribute__ ((__aligned__ (__alignof__ (md5_uint32)))); +}; + +/* + * The following three functions are build up the low level used in + * the functions `md5_stream' and `md5_buffer'. + */ + +/* Initialize structure containing state of computation. + (RFC 1321, 3.3: Step 3) */ +extern void MD5_Init (struct MD5_CTX *ctx); + +/* Starting with the result of former calls of this function (or the + initialization function update the context for the next LEN bytes + starting at BUFFER. + It is necessary that LEN is a multiple of 64!!! */ +extern void md5_process_block (const void *buffer, size_t len, + struct MD5_CTX *ctx); + +/* Starting with the result of former calls of this function (or the + initialization function update the context for the next LEN bytes + starting at BUFFER. + It is NOT required that LEN is a multiple of 64. */ +extern void MD5_Update (struct MD5_CTX *ctx, const void *buffer, size_t len); + +/* Process the remaining bytes in the buffer and put result from CTX + in first 16 bytes following RESBUF. The result is always in little + endian byte order, so that a byte-wise output yields to the wanted + ASCII representation of the message digest. + + IMPORTANT: On some systems it is required that RESBUF is correctly + aligned for a 32 bits value. */ +extern void *MD5_Final (void *resbuf, struct MD5_CTX *ctx); + + +/* Put result from CTX in first 16 bytes following RESBUF. The result is + always in little endian byte order, so that a byte-wise output yields + to the wanted ASCII representation of the message digest. + + IMPORTANT: On some systems it is required that RESBUF is correctly + aligned for a 32 bits value. */ +extern void *md5_read_ctx (const struct MD5_CTX *ctx, void *resbuf); + + +#endif /* md5.h */ diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc new file mode 100644 index 000000000000..6b71f52c151b --- /dev/null +++ b/src/libutil/serialise.cc @@ -0,0 +1,259 @@ +#include "serialise.hh" +#include "util.hh" + +#include <cstring> +#include <cerrno> + + +namespace nix { + + +BufferedSink::~BufferedSink() +{ + /* We can't call flush() here, because C++ for some insane reason + doesn't allow you to call virtual methods from a destructor. */ + assert(!bufPos); + delete[] buffer; +} + + +void BufferedSink::operator () (const unsigned char * data, size_t len) +{ + if (!buffer) buffer = new unsigned char[bufSize]; + + while (len) { + /* Optimisation: bypass the buffer if the data exceeds the + buffer size. */ + if (bufPos + len >= bufSize) { + flush(); + write(data, len); + break; + } + /* Otherwise, copy the bytes to the buffer. Flush the buffer + when it's full. */ + size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; + memcpy(buffer + bufPos, data, n); + data += n; bufPos += n; len -= n; + if (bufPos == bufSize) flush(); + } +} + + +void BufferedSink::flush() +{ + if (bufPos == 0) return; + size_t n = bufPos; + bufPos = 0; // don't trigger the assert() in ~BufferedSink() + write(buffer, n); +} + + +FdSink::~FdSink() +{ + try { flush(); } catch (...) { ignoreException(); } +} + + +void FdSink::write(const unsigned char * data, size_t len) +{ + writeFull(fd, data, len); +} + + +void Source::operator () (unsigned char * data, size_t len) +{ + while (len) { + size_t n = read(data, len); + data += n; len -= n; + } +} + + +BufferedSource::~BufferedSource() +{ + delete[] buffer; +} + + +size_t BufferedSource::read(unsigned char * data, size_t len) +{ + if (!buffer) buffer = new unsigned char[bufSize]; + + if (!bufPosIn) bufPosIn = readUnbuffered(buffer, bufSize); + + /* Copy out the data in the buffer. */ + size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; + memcpy(data, buffer + bufPosOut, n); + bufPosOut += n; + if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; + return n; +} + + +bool BufferedSource::hasData() +{ + return bufPosOut < bufPosIn; +} + + +size_t FdSource::readUnbuffered(unsigned char * data, size_t len) +{ + ssize_t n; + do { + checkInterrupt(); + n = ::read(fd, (char *) data, bufSize); + } while (n == -1 && errno == EINTR); + if (n == -1) throw SysError("reading from file"); + if (n == 0) throw EndOfFile("unexpected end-of-file"); + return n; +} + + +size_t StringSource::read(unsigned char * data, size_t len) +{ + if (pos == s.size()) throw EndOfFile("end of string reached"); + size_t n = s.copy((char *) data, len, pos); + pos += n; + return n; +} + + +void writePadding(size_t len, Sink & sink) +{ + if (len % 8) { + unsigned char zero[8]; + memset(zero, 0, sizeof(zero)); + sink(zero, 8 - (len % 8)); + } +} + + +void writeInt(unsigned int n, Sink & sink) +{ + unsigned char buf[8]; + memset(buf, 0, sizeof(buf)); + buf[0] = n & 0xff; + buf[1] = (n >> 8) & 0xff; + buf[2] = (n >> 16) & 0xff; + buf[3] = (n >> 24) & 0xff; + sink(buf, sizeof(buf)); +} + + +void writeLongLong(unsigned long long n, Sink & sink) +{ + unsigned char buf[8]; + buf[0] = n & 0xff; + buf[1] = (n >> 8) & 0xff; + buf[2] = (n >> 16) & 0xff; + buf[3] = (n >> 24) & 0xff; + buf[4] = (n >> 32) & 0xff; + buf[5] = (n >> 40) & 0xff; + buf[6] = (n >> 48) & 0xff; + buf[7] = (n >> 56) & 0xff; + sink(buf, sizeof(buf)); +} + + +void writeString(const unsigned char * buf, size_t len, Sink & sink) +{ + writeInt(len, sink); + sink(buf, len); + writePadding(len, sink); +} + + +void writeString(const string & s, Sink & sink) +{ + writeString((const unsigned char *) s.data(), s.size(), sink); +} + + +template<class T> void writeStrings(const T & ss, Sink & sink) +{ + writeInt(ss.size(), sink); + foreach (typename T::const_iterator, i, ss) + writeString(*i, sink); +} + +template void writeStrings(const Paths & ss, Sink & sink); +template void writeStrings(const PathSet & ss, Sink & sink); + + +void readPadding(size_t len, Source & source) +{ + if (len % 8) { + unsigned char zero[8]; + size_t n = 8 - (len % 8); + source(zero, n); + for (unsigned int i = 0; i < n; i++) + if (zero[i]) throw SerialisationError("non-zero padding"); + } +} + + +unsigned int readInt(Source & source) +{ + unsigned char buf[8]; + source(buf, sizeof(buf)); + if (buf[4] || buf[5] || buf[6] || buf[7]) + throw SerialisationError("implementation cannot deal with > 32-bit integers"); + return + buf[0] | + (buf[1] << 8) | + (buf[2] << 16) | + (buf[3] << 24); +} + + +unsigned long long readLongLong(Source & source) +{ + unsigned char buf[8]; + source(buf, sizeof(buf)); + return + ((unsigned long long) buf[0]) | + ((unsigned long long) buf[1] << 8) | + ((unsigned long long) buf[2] << 16) | + ((unsigned long long) buf[3] << 24) | + ((unsigned long long) buf[4] << 32) | + ((unsigned long long) buf[5] << 40) | + ((unsigned long long) buf[6] << 48) | + ((unsigned long long) buf[7] << 56); +} + + +size_t readString(unsigned char * buf, size_t max, Source & source) +{ + size_t len = readInt(source); + if (len > max) throw Error("string is too long"); + source(buf, len); + readPadding(len, source); + return len; +} + + +string readString(Source & source) +{ + size_t len = readInt(source); + unsigned char * buf = new unsigned char[len]; + AutoDeleteArray<unsigned char> d(buf); + source(buf, len); + readPadding(len, source); + return string((char *) buf, len); +} + + +template<class T> T readStrings(Source & source) +{ + unsigned int count = readInt(source); + T ss; + while (count--) + ss.insert(ss.end(), readString(source)); + return ss; +} + +template Paths readStrings(Source & source); +template PathSet readStrings(Source & source); + + +} diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh new file mode 100644 index 000000000000..e5a9df1d050a --- /dev/null +++ b/src/libutil/serialise.hh @@ -0,0 +1,133 @@ +#pragma once + +#include "types.hh" + + +namespace nix { + + +/* Abstract destination of binary data. */ +struct Sink +{ + virtual ~Sink() { } + virtual void operator () (const unsigned char * data, size_t len) = 0; +}; + + +/* A buffered abstract sink. */ +struct BufferedSink : Sink +{ + size_t bufSize, bufPos; + unsigned char * buffer; + + BufferedSink(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPos(0), buffer(0) { } + ~BufferedSink(); + + void operator () (const unsigned char * data, size_t len); + + void flush(); + + virtual void write(const unsigned char * data, size_t len) = 0; +}; + + +/* Abstract source of binary data. */ +struct Source +{ + virtual ~Source() { } + + /* Store exactly โlenโ bytes in the buffer pointed to by โdataโ. + It blocks until all the requested data is available, or throws + an error if it is not going to be available. */ + void operator () (unsigned char * data, size_t len); + + /* Store up to โlenโ in the buffer pointed to by โdataโ, and + return the number of bytes stored. If blocks until at least + one byte is available. */ + virtual size_t read(unsigned char * data, size_t len) = 0; +}; + + +/* A buffered abstract source. */ +struct BufferedSource : Source +{ + size_t bufSize, bufPosIn, bufPosOut; + unsigned char * buffer; + + BufferedSource(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } + ~BufferedSource(); + + size_t read(unsigned char * data, size_t len); + + /* Underlying read call, to be overridden. */ + virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; + + bool hasData(); +}; + + +/* A sink that writes data to a file descriptor. */ +struct FdSink : BufferedSink +{ + int fd; + + FdSink() : fd(-1) { } + FdSink(int fd) : fd(fd) { } + ~FdSink(); + + void write(const unsigned char * data, size_t len); +}; + + +/* A source that reads data from a file descriptor. */ +struct FdSource : BufferedSource +{ + int fd; + FdSource() : fd(-1) { } + FdSource(int fd) : fd(fd) { } + size_t readUnbuffered(unsigned char * data, size_t len); +}; + + +/* A sink that writes data to a string. */ +struct StringSink : Sink +{ + string s; + void operator () (const unsigned char * data, size_t len) + { + s.append((const char *) data, len); + } +}; + + +/* A source that reads data from a string. */ +struct StringSource : Source +{ + const string & s; + size_t pos; + StringSource(const string & _s) : s(_s), pos(0) { } + size_t read(unsigned char * data, size_t len); +}; + + +void writePadding(size_t len, Sink & sink); +void writeInt(unsigned int n, Sink & sink); +void writeLongLong(unsigned long long n, Sink & sink); +void writeString(const unsigned char * buf, size_t len, Sink & sink); +void writeString(const string & s, Sink & sink); +template<class T> void writeStrings(const T & ss, Sink & sink); + +void readPadding(size_t len, Source & source); +unsigned int readInt(Source & source); +unsigned long long readLongLong(Source & source); +size_t readString(unsigned char * buf, size_t max, Source & source); +string readString(Source & source); +template<class T> T readStrings(Source & source); + + +MakeError(SerialisationError, Error) + + +} diff --git a/src/libutil/sha1.c b/src/libutil/sha1.c new file mode 100644 index 000000000000..d9d294d15540 --- /dev/null +++ b/src/libutil/sha1.c @@ -0,0 +1,369 @@ +/* $Id$ */ + +/* sha.c - Implementation of the Secure Hash Algorithm + * + * Copyright (C) 1995, A.M. Kuchling + * + * Distribute and use freely; there are no restrictions on further + * dissemination and usage except those imposed by the laws of your + * country of residence. + * + * Adapted to pike and some cleanup by Niels M๖ller. + */ + +/* $Id$ */ + +/* SHA: NIST's Secure Hash Algorithm */ + +/* Based on SHA code originally posted to sci.crypt by Peter Gutmann + in message <30ajo5$oe8@ccu2.auckland.ac.nz>. + Modified to test for endianness on creation of SHA objects by AMK. + Also, the original specification of SHA was found to have a weakness + by NSA/NIST. This code implements the fixed version of SHA. +*/ + +/* Here's the first paragraph of Peter Gutmann's posting: + +The following is my SHA (FIPS 180) code updated to allow use of the "fixed" +SHA, thanks to Jim Gillogly and an anonymous contributor for the information on +what's changed in the new version. The fix is a simple change which involves +adding a single rotate in the initial expansion function. It is unknown +whether this is an optimal solution to the problem which was discovered in the +SHA or whether it's simply a bandaid which fixes the problem with a minimum of +effort (for example the reengineering of a great many Capstone chips). +*/ + +#include "sha1.h" + +#include <string.h> + +void sha_copy(struct SHA_CTX *dest, struct SHA_CTX *src) +{ + unsigned int i; + + dest->count_l=src->count_l; + dest->count_h=src->count_h; + for(i=0; i<SHA_DIGESTLEN; i++) + dest->digest[i]=src->digest[i]; + for(i=0; i < src->index; i++) + dest->block[i] = src->block[i]; + dest->index = src->index; +} + + +/* The SHA f()-functions. The f1 and f3 functions can be optimized to + save one boolean operation each - thanks to Rich Schroeppel, + rcs@cs.arizona.edu for discovering this */ + +/*#define f1(x,y,z) ( ( x & y ) | ( ~x & z ) ) // Rounds 0-19 */ +#define f1(x,y,z) ( z ^ ( x & ( y ^ z ) ) ) /* Rounds 0-19 */ +#define f2(x,y,z) ( x ^ y ^ z ) /* Rounds 20-39 */ +/*#define f3(x,y,z) ( ( x & y ) | ( x & z ) | ( y & z ) ) // Rounds 40-59 */ +#define f3(x,y,z) ( ( x & y ) | ( z & ( x | y ) ) ) /* Rounds 40-59 */ +#define f4(x,y,z) ( x ^ y ^ z ) /* Rounds 60-79 */ + +/* The SHA Mysterious Constants */ + +#define K1 0x5A827999L /* Rounds 0-19 */ +#define K2 0x6ED9EBA1L /* Rounds 20-39 */ +#define K3 0x8F1BBCDCL /* Rounds 40-59 */ +#define K4 0xCA62C1D6L /* Rounds 60-79 */ + +/* SHA initial values */ + +#define h0init 0x67452301L +#define h1init 0xEFCDAB89L +#define h2init 0x98BADCFEL +#define h3init 0x10325476L +#define h4init 0xC3D2E1F0L + +/* 32-bit rotate left - kludged with shifts */ + +#define ROTL(n,X) ( ( (X) << (n) ) | ( (X) >> ( 32 - (n) ) ) ) + +/* The initial expanding function. The hash function is defined over an + 80-word expanded input array W, where the first 16 are copies of the input + data, and the remaining 64 are defined by + + W[ i ] = W[ i - 16 ] ^ W[ i - 14 ] ^ W[ i - 8 ] ^ W[ i - 3 ] + + This implementation generates these values on the fly in a circular + buffer - thanks to Colin Plumb, colin@nyx10.cs.du.edu for this + optimization. + + The updated SHA changes the expanding function by adding a rotate of 1 + bit. Thanks to Jim Gillogly, jim@rand.org, and an anonymous contributor + for this information */ + +#define expand(W,i) ( W[ i & 15 ] = \ + ROTL( 1, ( W[ i & 15 ] ^ W[ (i - 14) & 15 ] ^ \ + W[ (i - 8) & 15 ] ^ W[ (i - 3) & 15 ] ) ) ) + + +/* The prototype SHA sub-round. The fundamental sub-round is: + + a' = e + ROTL( 5, a ) + f( b, c, d ) + k + data; + b' = a; + c' = ROTL( 30, b ); + d' = c; + e' = d; + + but this is implemented by unrolling the loop 5 times and renaming the + variables ( e, a, b, c, d ) = ( a', b', c', d', e' ) each iteration. + This code is then replicated 20 times for each of the 4 functions, using + the next 20 values from the W[] array each time */ + +#define subRound(a, b, c, d, e, f, k, data) \ + ( e += ROTL( 5, a ) + f( b, c, d ) + k + data, b = ROTL( 30, b ) ) + +/* Initialize the SHA values */ + +void SHA1_Init(struct SHA_CTX *ctx) +{ + /* Set the h-vars to their initial values */ + ctx->digest[ 0 ] = h0init; + ctx->digest[ 1 ] = h1init; + ctx->digest[ 2 ] = h2init; + ctx->digest[ 3 ] = h3init; + ctx->digest[ 4 ] = h4init; + + /* Initialize bit count */ + ctx->count_l = ctx->count_h = 0; + + /* Initialize buffer */ + ctx->index = 0; +} + +/* Perform the SHA transformation. Note that this code, like MD5, seems to + break some optimizing compilers due to the complexity of the expressions + and the size of the basic block. It may be necessary to split it into + sections, e.g. based on the four subrounds + + Note that this function destroys the data area */ + +static void sha_transform(struct SHA_CTX *ctx, uint32_t *data ) +{ + uint32_t A, B, C, D, E; /* Local vars */ + + /* Set up first buffer and local data buffer */ + A = ctx->digest[0]; + B = ctx->digest[1]; + C = ctx->digest[2]; + D = ctx->digest[3]; + E = ctx->digest[4]; + + /* Heavy mangling, in 4 sub-rounds of 20 interations each. */ + subRound( A, B, C, D, E, f1, K1, data[ 0] ); + subRound( E, A, B, C, D, f1, K1, data[ 1] ); + subRound( D, E, A, B, C, f1, K1, data[ 2] ); + subRound( C, D, E, A, B, f1, K1, data[ 3] ); + subRound( B, C, D, E, A, f1, K1, data[ 4] ); + subRound( A, B, C, D, E, f1, K1, data[ 5] ); + subRound( E, A, B, C, D, f1, K1, data[ 6] ); + subRound( D, E, A, B, C, f1, K1, data[ 7] ); + subRound( C, D, E, A, B, f1, K1, data[ 8] ); + subRound( B, C, D, E, A, f1, K1, data[ 9] ); + subRound( A, B, C, D, E, f1, K1, data[10] ); + subRound( E, A, B, C, D, f1, K1, data[11] ); + subRound( D, E, A, B, C, f1, K1, data[12] ); + subRound( C, D, E, A, B, f1, K1, data[13] ); + subRound( B, C, D, E, A, f1, K1, data[14] ); + subRound( A, B, C, D, E, f1, K1, data[15] ); + subRound( E, A, B, C, D, f1, K1, expand( data, 16 ) ); + subRound( D, E, A, B, C, f1, K1, expand( data, 17 ) ); + subRound( C, D, E, A, B, f1, K1, expand( data, 18 ) ); + subRound( B, C, D, E, A, f1, K1, expand( data, 19 ) ); + + subRound( A, B, C, D, E, f2, K2, expand( data, 20 ) ); + subRound( E, A, B, C, D, f2, K2, expand( data, 21 ) ); + subRound( D, E, A, B, C, f2, K2, expand( data, 22 ) ); + subRound( C, D, E, A, B, f2, K2, expand( data, 23 ) ); + subRound( B, C, D, E, A, f2, K2, expand( data, 24 ) ); + subRound( A, B, C, D, E, f2, K2, expand( data, 25 ) ); + subRound( E, A, B, C, D, f2, K2, expand( data, 26 ) ); + subRound( D, E, A, B, C, f2, K2, expand( data, 27 ) ); + subRound( C, D, E, A, B, f2, K2, expand( data, 28 ) ); + subRound( B, C, D, E, A, f2, K2, expand( data, 29 ) ); + subRound( A, B, C, D, E, f2, K2, expand( data, 30 ) ); + subRound( E, A, B, C, D, f2, K2, expand( data, 31 ) ); + subRound( D, E, A, B, C, f2, K2, expand( data, 32 ) ); + subRound( C, D, E, A, B, f2, K2, expand( data, 33 ) ); + subRound( B, C, D, E, A, f2, K2, expand( data, 34 ) ); + subRound( A, B, C, D, E, f2, K2, expand( data, 35 ) ); + subRound( E, A, B, C, D, f2, K2, expand( data, 36 ) ); + subRound( D, E, A, B, C, f2, K2, expand( data, 37 ) ); + subRound( C, D, E, A, B, f2, K2, expand( data, 38 ) ); + subRound( B, C, D, E, A, f2, K2, expand( data, 39 ) ); + + subRound( A, B, C, D, E, f3, K3, expand( data, 40 ) ); + subRound( E, A, B, C, D, f3, K3, expand( data, 41 ) ); + subRound( D, E, A, B, C, f3, K3, expand( data, 42 ) ); + subRound( C, D, E, A, B, f3, K3, expand( data, 43 ) ); + subRound( B, C, D, E, A, f3, K3, expand( data, 44 ) ); + subRound( A, B, C, D, E, f3, K3, expand( data, 45 ) ); + subRound( E, A, B, C, D, f3, K3, expand( data, 46 ) ); + subRound( D, E, A, B, C, f3, K3, expand( data, 47 ) ); + subRound( C, D, E, A, B, f3, K3, expand( data, 48 ) ); + subRound( B, C, D, E, A, f3, K3, expand( data, 49 ) ); + subRound( A, B, C, D, E, f3, K3, expand( data, 50 ) ); + subRound( E, A, B, C, D, f3, K3, expand( data, 51 ) ); + subRound( D, E, A, B, C, f3, K3, expand( data, 52 ) ); + subRound( C, D, E, A, B, f3, K3, expand( data, 53 ) ); + subRound( B, C, D, E, A, f3, K3, expand( data, 54 ) ); + subRound( A, B, C, D, E, f3, K3, expand( data, 55 ) ); + subRound( E, A, B, C, D, f3, K3, expand( data, 56 ) ); + subRound( D, E, A, B, C, f3, K3, expand( data, 57 ) ); + subRound( C, D, E, A, B, f3, K3, expand( data, 58 ) ); + subRound( B, C, D, E, A, f3, K3, expand( data, 59 ) ); + + subRound( A, B, C, D, E, f4, K4, expand( data, 60 ) ); + subRound( E, A, B, C, D, f4, K4, expand( data, 61 ) ); + subRound( D, E, A, B, C, f4, K4, expand( data, 62 ) ); + subRound( C, D, E, A, B, f4, K4, expand( data, 63 ) ); + subRound( B, C, D, E, A, f4, K4, expand( data, 64 ) ); + subRound( A, B, C, D, E, f4, K4, expand( data, 65 ) ); + subRound( E, A, B, C, D, f4, K4, expand( data, 66 ) ); + subRound( D, E, A, B, C, f4, K4, expand( data, 67 ) ); + subRound( C, D, E, A, B, f4, K4, expand( data, 68 ) ); + subRound( B, C, D, E, A, f4, K4, expand( data, 69 ) ); + subRound( A, B, C, D, E, f4, K4, expand( data, 70 ) ); + subRound( E, A, B, C, D, f4, K4, expand( data, 71 ) ); + subRound( D, E, A, B, C, f4, K4, expand( data, 72 ) ); + subRound( C, D, E, A, B, f4, K4, expand( data, 73 ) ); + subRound( B, C, D, E, A, f4, K4, expand( data, 74 ) ); + subRound( A, B, C, D, E, f4, K4, expand( data, 75 ) ); + subRound( E, A, B, C, D, f4, K4, expand( data, 76 ) ); + subRound( D, E, A, B, C, f4, K4, expand( data, 77 ) ); + subRound( C, D, E, A, B, f4, K4, expand( data, 78 ) ); + subRound( B, C, D, E, A, f4, K4, expand( data, 79 ) ); + + /* Build message digest */ + ctx->digest[0] += A; + ctx->digest[1] += B; + ctx->digest[2] += C; + ctx->digest[3] += D; + ctx->digest[4] += E; +} + +#if 1 + +#ifndef EXTRACT_UCHAR +#define EXTRACT_UCHAR(p) (*(unsigned char *)(p)) +#endif + +#define STRING2INT(s) ((((((EXTRACT_UCHAR(s) << 8) \ + | EXTRACT_UCHAR(s+1)) << 8) \ + | EXTRACT_UCHAR(s+2)) << 8) \ + | EXTRACT_UCHAR(s+3)) +#else +uint32_t STRING2INT(unsigned char *s) +{ + uint32_t r; + unsigned int i; + + for (i = 0, r = 0; i < 4; i++, s++) + r = (r << 8) | *s; + return r; +} +#endif + +static void sha_block(struct SHA_CTX *ctx, const unsigned char *block) +{ + uint32_t data[SHA_DATALEN]; + unsigned int i; + + /* Update block count */ + if (!++ctx->count_l) + ++ctx->count_h; + + /* Endian independent conversion */ + for (i = 0; i<SHA_DATALEN; i++, block += 4) + data[i] = STRING2INT(block); + + sha_transform(ctx, data); +} + +void SHA1_Update(struct SHA_CTX *ctx, const unsigned char *buffer, uint32_t len) +{ + if (ctx->index) + { /* Try to fill partial block */ + unsigned left = SHA_DATASIZE - ctx->index; + if (len < left) + { + memcpy(ctx->block + ctx->index, buffer, len); + ctx->index += len; + return; /* Finished */ + } + else + { + memcpy(ctx->block + ctx->index, buffer, left); + sha_block(ctx, ctx->block); + buffer += left; + len -= left; + } + } + while (len >= SHA_DATASIZE) + { + sha_block(ctx, buffer); + buffer += SHA_DATASIZE; + len -= SHA_DATASIZE; + } + if ((ctx->index = len)) /* This assignment is intended */ + /* Buffer leftovers */ + memcpy(ctx->block, buffer, len); +} + +/* Final wrapup - pad to SHA_DATASIZE-byte boundary with the bit pattern + 1 0* (64-bit count of bits processed, MSB-first) */ + +void SHA1_Final(unsigned char *s, struct SHA_CTX *ctx) +{ + uint32_t data[SHA_DATALEN]; + unsigned int i; + unsigned int words; + + i = ctx->index; + /* Set the first char of padding to 0x80. This is safe since there is + always at least one byte free */ + ctx->block[i++] = 0x80; + + /* Fill rest of word */ + for( ; i & 3; i++) + ctx->block[i] = 0; + + /* i is now a multiple of the word size 4 */ + words = i >> 2; + for (i = 0; i < words; i++) + data[i] = STRING2INT(ctx->block + 4*i); + + if (words > (SHA_DATALEN-2)) + { /* No room for length in this block. Process it and + * pad with another one */ + for (i = words ; i < SHA_DATALEN; i++) + data[i] = 0; + sha_transform(ctx, data); + for (i = 0; i < (SHA_DATALEN-2); i++) + data[i] = 0; + } + else + for (i = words ; i < SHA_DATALEN - 2; i++) + data[i] = 0; + /* Theres 512 = 2^9 bits in one block */ + data[SHA_DATALEN-2] = (ctx->count_h << 9) | (ctx->count_l >> 23); + data[SHA_DATALEN-1] = (ctx->count_l << 9) | (ctx->index << 3); + sha_transform(ctx, data); + sha_digest(ctx, s); +} + +void sha_digest(struct SHA_CTX *ctx, unsigned char *s) +{ + unsigned int i; + + for (i = 0; i < SHA_DIGESTLEN; i++) + { + *s++ = ctx->digest[i] >> 24; + *s++ = 0xff & (ctx->digest[i] >> 16); + *s++ = 0xff & (ctx->digest[i] >> 8); + *s++ = 0xff & ctx->digest[i]; + } +} diff --git a/src/libutil/sha1.h b/src/libutil/sha1.h new file mode 100644 index 000000000000..715040dd48df --- /dev/null +++ b/src/libutil/sha1.h @@ -0,0 +1,28 @@ +#ifndef _SHA_H +#define _SHA_H + +#include <inttypes.h> + +/* The SHA block size and message digest sizes, in bytes */ + +#define SHA_DATASIZE 64 +#define SHA_DATALEN 16 +#define SHA_DIGESTSIZE 20 +#define SHA_DIGESTLEN 5 +/* The structure for storing SHA info */ + +struct SHA_CTX { + uint32_t digest[SHA_DIGESTLEN]; /* Message digest */ + uint32_t count_l, count_h; /* 64-bit block count */ + uint8_t block[SHA_DATASIZE]; /* SHA data buffer */ + unsigned int index; /* index into buffer */ +}; + +void SHA1_Init(struct SHA_CTX *ctx); +void SHA1_Update(struct SHA_CTX *ctx, const unsigned char *buffer, uint32_t len); +void SHA1_Final(unsigned char *s, struct SHA_CTX *ctx); +void sha_digest(struct SHA_CTX *ctx, unsigned char *s); +void sha_copy(struct SHA_CTX *dest, struct SHA_CTX *src); + + +#endif /* !_SHA_H */ diff --git a/src/libutil/sha256.c b/src/libutil/sha256.c new file mode 100644 index 000000000000..63ed0ba43011 --- /dev/null +++ b/src/libutil/sha256.c @@ -0,0 +1,238 @@ +/* crypto/sha/sha256.c */ +/* ==================================================================== + * Copyright (c) 2004 The OpenSSL Project. All rights reserved + * according to the OpenSSL license [found in ./md32_common.h]. + * ==================================================================== + */ + +#include <stdlib.h> +#include <string.h> + +#include "sha256.h" + +int SHA224_Init (SHA256_CTX *c) + { + c->h[0]=0xc1059ed8UL; c->h[1]=0x367cd507UL; + c->h[2]=0x3070dd17UL; c->h[3]=0xf70e5939UL; + c->h[4]=0xffc00b31UL; c->h[5]=0x68581511UL; + c->h[6]=0x64f98fa7UL; c->h[7]=0xbefa4fa4UL; + c->Nl=0; c->Nh=0; + c->num=0; c->md_len=SHA224_DIGEST_LENGTH; + return 1; + } + +int SHA256_Init (SHA256_CTX *c) + { + c->h[0]=0x6a09e667UL; c->h[1]=0xbb67ae85UL; + c->h[2]=0x3c6ef372UL; c->h[3]=0xa54ff53aUL; + c->h[4]=0x510e527fUL; c->h[5]=0x9b05688cUL; + c->h[6]=0x1f83d9abUL; c->h[7]=0x5be0cd19UL; + c->Nl=0; c->Nh=0; + c->num=0; c->md_len=SHA256_DIGEST_LENGTH; + return 1; + } + +unsigned char *SHA224(const unsigned char *d, size_t n, unsigned char *md) + { + SHA256_CTX c; + static unsigned char m[SHA224_DIGEST_LENGTH]; + + if (md == NULL) md=m; + SHA224_Init(&c); + SHA256_Update(&c,d,n); + SHA256_Final(md,&c); + return(md); + } + +unsigned char *SHA256(const unsigned char *d, size_t n, unsigned char *md) + { + SHA256_CTX c; + static unsigned char m[SHA256_DIGEST_LENGTH]; + + if (md == NULL) md=m; + SHA256_Init(&c); + SHA256_Update(&c,d,n); + SHA256_Final(md,&c); + return(md); + } + +int SHA224_Update(SHA256_CTX *c, const void *data, size_t len) +{ return SHA256_Update (c,data,len); } +int SHA224_Final (unsigned char *md, SHA256_CTX *c) +{ return SHA256_Final (md,c); } + +#define DATA_ORDER_IS_BIG_ENDIAN + +#define HASH_LONG uint32_t +#define HASH_LONG_LOG2 2 +#define HASH_CTX SHA256_CTX +#define HASH_CBLOCK SHA_CBLOCK +#define HASH_LBLOCK SHA_LBLOCK +/* + * Note that FIPS180-2 discusses "Truncation of the Hash Function Output." + * default: case below covers for it. It's not clear however if it's + * permitted to truncate to amount of bytes not divisible by 4. I bet not, + * but if it is, then default: case shall be extended. For reference. + * Idea behind separate cases for pre-defined lenghts is to let the + * compiler decide if it's appropriate to unroll small loops. + */ +#define HASH_MAKE_STRING(c,s) do { \ + unsigned long ll; \ + unsigned int n; \ + switch ((c)->md_len) \ + { case SHA224_DIGEST_LENGTH: \ + for (n=0;n<SHA224_DIGEST_LENGTH/4;n++) \ + { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ + break; \ + case SHA256_DIGEST_LENGTH: \ + for (n=0;n<SHA256_DIGEST_LENGTH/4;n++) \ + { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ + break; \ + default: \ + if ((c)->md_len > SHA256_DIGEST_LENGTH) \ + return 0; \ + for (n=0;n<(c)->md_len/4;n++) \ + { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ + break; \ + } \ + } while (0) + +#define HASH_UPDATE SHA256_Update +#define HASH_TRANSFORM SHA256_Transform +#define HASH_FINAL SHA256_Final +#define HASH_BLOCK_HOST_ORDER sha256_block_host_order +#define HASH_BLOCK_DATA_ORDER sha256_block_data_order +void sha256_block_host_order (SHA256_CTX *ctx, const void *in, size_t num); +void sha256_block_data_order (SHA256_CTX *ctx, const void *in, size_t num); + +#include "md32_common.h" + +static const uint32_t K256[64] = { + 0x428a2f98UL,0x71374491UL,0xb5c0fbcfUL,0xe9b5dba5UL, + 0x3956c25bUL,0x59f111f1UL,0x923f82a4UL,0xab1c5ed5UL, + 0xd807aa98UL,0x12835b01UL,0x243185beUL,0x550c7dc3UL, + 0x72be5d74UL,0x80deb1feUL,0x9bdc06a7UL,0xc19bf174UL, + 0xe49b69c1UL,0xefbe4786UL,0x0fc19dc6UL,0x240ca1ccUL, + 0x2de92c6fUL,0x4a7484aaUL,0x5cb0a9dcUL,0x76f988daUL, + 0x983e5152UL,0xa831c66dUL,0xb00327c8UL,0xbf597fc7UL, + 0xc6e00bf3UL,0xd5a79147UL,0x06ca6351UL,0x14292967UL, + 0x27b70a85UL,0x2e1b2138UL,0x4d2c6dfcUL,0x53380d13UL, + 0x650a7354UL,0x766a0abbUL,0x81c2c92eUL,0x92722c85UL, + 0xa2bfe8a1UL,0xa81a664bUL,0xc24b8b70UL,0xc76c51a3UL, + 0xd192e819UL,0xd6990624UL,0xf40e3585UL,0x106aa070UL, + 0x19a4c116UL,0x1e376c08UL,0x2748774cUL,0x34b0bcb5UL, + 0x391c0cb3UL,0x4ed8aa4aUL,0x5b9cca4fUL,0x682e6ff3UL, + 0x748f82eeUL,0x78a5636fUL,0x84c87814UL,0x8cc70208UL, + 0x90befffaUL,0xa4506cebUL,0xbef9a3f7UL,0xc67178f2UL }; + +/* + * FIPS specification refers to right rotations, while our ROTATE macro + * is left one. This is why you might notice that rotation coefficients + * differ from those observed in FIPS document by 32-N... + */ +#define Sigma0(x) (ROTATE((x),30) ^ ROTATE((x),19) ^ ROTATE((x),10)) +#define Sigma1(x) (ROTATE((x),26) ^ ROTATE((x),21) ^ ROTATE((x),7)) +#define sigma0(x) (ROTATE((x),25) ^ ROTATE((x),14) ^ ((x)>>3)) +#define sigma1(x) (ROTATE((x),15) ^ ROTATE((x),13) ^ ((x)>>10)) + +#define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) +#define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) + +#define ROUND_00_15(i,a,b,c,d,e,f,g,h) do { \ + T1 += h + Sigma1(e) + Ch(e,f,g) + K256[i]; \ + h = Sigma0(a) + Maj(a,b,c); \ + d += T1; h += T1; } while (0) + +#define ROUND_16_63(i,a,b,c,d,e,f,g,h,X) do { \ + s0 = X[(i+1)&0x0f]; s0 = sigma0(s0); \ + s1 = X[(i+14)&0x0f]; s1 = sigma1(s1); \ + T1 = X[(i)&0x0f] += s0 + s1 + X[(i+9)&0x0f]; \ + ROUND_00_15(i,a,b,c,d,e,f,g,h); } while (0) + +static void sha256_block (SHA256_CTX *ctx, const void *in, size_t num, int host) + { + uint32_t a,b,c,d,e,f,g,h,s0,s1,T1; + uint32_t X[16]; + int i; + const unsigned char *data=in; + + while (num--) { + + a = ctx->h[0]; b = ctx->h[1]; c = ctx->h[2]; d = ctx->h[3]; + e = ctx->h[4]; f = ctx->h[5]; g = ctx->h[6]; h = ctx->h[7]; + + if (host) + { + const uint32_t *W=(const uint32_t *)data; + + T1 = X[0] = W[0]; ROUND_00_15(0,a,b,c,d,e,f,g,h); + T1 = X[1] = W[1]; ROUND_00_15(1,h,a,b,c,d,e,f,g); + T1 = X[2] = W[2]; ROUND_00_15(2,g,h,a,b,c,d,e,f); + T1 = X[3] = W[3]; ROUND_00_15(3,f,g,h,a,b,c,d,e); + T1 = X[4] = W[4]; ROUND_00_15(4,e,f,g,h,a,b,c,d); + T1 = X[5] = W[5]; ROUND_00_15(5,d,e,f,g,h,a,b,c); + T1 = X[6] = W[6]; ROUND_00_15(6,c,d,e,f,g,h,a,b); + T1 = X[7] = W[7]; ROUND_00_15(7,b,c,d,e,f,g,h,a); + T1 = X[8] = W[8]; ROUND_00_15(8,a,b,c,d,e,f,g,h); + T1 = X[9] = W[9]; ROUND_00_15(9,h,a,b,c,d,e,f,g); + T1 = X[10] = W[10]; ROUND_00_15(10,g,h,a,b,c,d,e,f); + T1 = X[11] = W[11]; ROUND_00_15(11,f,g,h,a,b,c,d,e); + T1 = X[12] = W[12]; ROUND_00_15(12,e,f,g,h,a,b,c,d); + T1 = X[13] = W[13]; ROUND_00_15(13,d,e,f,g,h,a,b,c); + T1 = X[14] = W[14]; ROUND_00_15(14,c,d,e,f,g,h,a,b); + T1 = X[15] = W[15]; ROUND_00_15(15,b,c,d,e,f,g,h,a); + + data += SHA256_CBLOCK; + } + else + { + uint32_t l; + + HOST_c2l(data,l); T1 = X[0] = l; ROUND_00_15(0,a,b,c,d,e,f,g,h); + HOST_c2l(data,l); T1 = X[1] = l; ROUND_00_15(1,h,a,b,c,d,e,f,g); + HOST_c2l(data,l); T1 = X[2] = l; ROUND_00_15(2,g,h,a,b,c,d,e,f); + HOST_c2l(data,l); T1 = X[3] = l; ROUND_00_15(3,f,g,h,a,b,c,d,e); + HOST_c2l(data,l); T1 = X[4] = l; ROUND_00_15(4,e,f,g,h,a,b,c,d); + HOST_c2l(data,l); T1 = X[5] = l; ROUND_00_15(5,d,e,f,g,h,a,b,c); + HOST_c2l(data,l); T1 = X[6] = l; ROUND_00_15(6,c,d,e,f,g,h,a,b); + HOST_c2l(data,l); T1 = X[7] = l; ROUND_00_15(7,b,c,d,e,f,g,h,a); + HOST_c2l(data,l); T1 = X[8] = l; ROUND_00_15(8,a,b,c,d,e,f,g,h); + HOST_c2l(data,l); T1 = X[9] = l; ROUND_00_15(9,h,a,b,c,d,e,f,g); + HOST_c2l(data,l); T1 = X[10] = l; ROUND_00_15(10,g,h,a,b,c,d,e,f); + HOST_c2l(data,l); T1 = X[11] = l; ROUND_00_15(11,f,g,h,a,b,c,d,e); + HOST_c2l(data,l); T1 = X[12] = l; ROUND_00_15(12,e,f,g,h,a,b,c,d); + HOST_c2l(data,l); T1 = X[13] = l; ROUND_00_15(13,d,e,f,g,h,a,b,c); + HOST_c2l(data,l); T1 = X[14] = l; ROUND_00_15(14,c,d,e,f,g,h,a,b); + HOST_c2l(data,l); T1 = X[15] = l; ROUND_00_15(15,b,c,d,e,f,g,h,a); + } + + for (i=16;i<64;i+=8) + { + ROUND_16_63(i+0,a,b,c,d,e,f,g,h,X); + ROUND_16_63(i+1,h,a,b,c,d,e,f,g,X); + ROUND_16_63(i+2,g,h,a,b,c,d,e,f,X); + ROUND_16_63(i+3,f,g,h,a,b,c,d,e,X); + ROUND_16_63(i+4,e,f,g,h,a,b,c,d,X); + ROUND_16_63(i+5,d,e,f,g,h,a,b,c,X); + ROUND_16_63(i+6,c,d,e,f,g,h,a,b,X); + ROUND_16_63(i+7,b,c,d,e,f,g,h,a,X); + } + + ctx->h[0] += a; ctx->h[1] += b; ctx->h[2] += c; ctx->h[3] += d; + ctx->h[4] += e; ctx->h[5] += f; ctx->h[6] += g; ctx->h[7] += h; + + } + } + +/* + * Idea is to trade couple of cycles for some space. On IA-32 we save + * about 4K in "big footprint" case. In "small footprint" case any gain + * is appreciated:-) + */ +void HASH_BLOCK_HOST_ORDER (SHA256_CTX *ctx, const void *in, size_t num) +{ sha256_block (ctx,in,num,1); } + +void HASH_BLOCK_DATA_ORDER (SHA256_CTX *ctx, const void *in, size_t num) +{ sha256_block (ctx,in,num,0); } + + diff --git a/src/libutil/sha256.h b/src/libutil/sha256.h new file mode 100644 index 000000000000..0686b84f0e08 --- /dev/null +++ b/src/libutil/sha256.h @@ -0,0 +1,35 @@ +#ifndef _SHA256_H +#define _SHA256_H 1 + +#include <inttypes.h> + +#define SHA_LBLOCK 16 +#define SHA_CBLOCK (SHA_LBLOCK*4) /* SHA treats input data as a + * contiguous array of 32 bit + * wide big-endian values. */ + +#define SHA256_CBLOCK (SHA_LBLOCK*4) /* SHA-256 treats input data as a + * contiguous array of 32 bit + * wide big-endian values. */ +#define SHA224_DIGEST_LENGTH 28 +#define SHA256_DIGEST_LENGTH 32 + +typedef struct SHA256state_st + { + uint32_t h[8]; + uint32_t Nl,Nh; + uint32_t data[SHA_LBLOCK]; + unsigned int num,md_len; + } SHA256_CTX; + +int SHA224_Init(SHA256_CTX *c); +int SHA224_Update(SHA256_CTX *c, const void *data, size_t len); +int SHA224_Final(unsigned char *md, SHA256_CTX *c); +unsigned char *SHA224(const unsigned char *d, size_t n,unsigned char *md); +int SHA256_Init(SHA256_CTX *c); +int SHA256_Update(SHA256_CTX *c, const void *data, size_t len); +int SHA256_Final(unsigned char *md, SHA256_CTX *c); +unsigned char *SHA256(const unsigned char *d, size_t n,unsigned char *md); +void SHA256_Transform(SHA256_CTX *c, const unsigned char *data); + +#endif diff --git a/src/libutil/types.hh b/src/libutil/types.hh new file mode 100644 index 000000000000..4b5ce9a78c48 --- /dev/null +++ b/src/libutil/types.hh @@ -0,0 +1,86 @@ +#pragma once + +#include "config.h" + +#include <string> +#include <list> +#include <set> + +#include <boost/format.hpp> + + +namespace nix { + + +/* Inherit some names from other namespaces for convenience. */ +using std::string; +using std::list; +using std::set; +using std::vector; +using boost::format; + + +struct FormatOrString +{ + string s; + FormatOrString(const string & s) : s(s) { }; + FormatOrString(const format & f) : s(f.str()) { }; + FormatOrString(const char * s) : s(s) { }; +}; + + +/* BaseError should generally not be caught, as it has Interrupted as + a subclass. Catch Error instead. */ +class BaseError : public std::exception +{ +protected: + string prefix_; // used for location traces etc. + string err; +public: + unsigned int status; // exit status + BaseError(const FormatOrString & fs, unsigned int status = 1); + ~BaseError() throw () { }; + const char * what() const throw () { return err.c_str(); } + const string & msg() const throw () { return err; } + const string & prefix() const throw () { return prefix_; } + BaseError & addPrefix(const FormatOrString & fs); +}; + +#define MakeError(newClass, superClass) \ + class newClass : public superClass \ + { \ + public: \ + newClass(const FormatOrString & fs, unsigned int status = 1) : superClass(fs, status) { }; \ + }; + +MakeError(Error, BaseError) + +class SysError : public Error +{ +public: + int errNo; + SysError(const FormatOrString & fs); +}; + + +typedef list<string> Strings; +typedef set<string> StringSet; + + +/* Paths are just strings. */ +typedef string Path; +typedef list<Path> Paths; +typedef set<Path> PathSet; + + +typedef enum { + lvlError = 0, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit +} Verbosity; + + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc new file mode 100644 index 000000000000..15c462ce4e5b --- /dev/null +++ b/src/libutil/util.cc @@ -0,0 +1,1105 @@ +#include "config.h" + +#include <iostream> +#include <cerrno> +#include <cstdio> +#include <cstdlib> +#include <sstream> +#include <cstring> + +#include <sys/wait.h> +#include <unistd.h> +#include <fcntl.h> +#include <limits.h> + +#ifdef __APPLE__ +#include <sys/syscall.h> +#endif + +#include "util.hh" + + +extern char * * environ; + + +namespace nix { + + +BaseError::BaseError(const FormatOrString & fs, unsigned int status) + : status(status) +{ + err = fs.s; +} + + +BaseError & BaseError::addPrefix(const FormatOrString & fs) +{ + prefix_ = fs.s + prefix_; + return *this; +} + + +SysError::SysError(const FormatOrString & fs) + : Error(format("%1%: %2%") % fs.s % strerror(errno)) + , errNo(errno) +{ +} + + +string getEnv(const string & key, const string & def) +{ + char * value = getenv(key.c_str()); + return value ? string(value) : def; +} + + +Path absPath(Path path, Path dir) +{ + if (path[0] != '/') { + if (dir == "") { +#ifdef __GNU__ + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char *buf = getcwd(NULL, 0); + if (buf == NULL) +#else + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) +#endif + throw SysError("cannot get cwd"); + dir = buf; +#ifdef __GNU__ + free(buf); +#endif + } + path = dir + "/" + path; + } + return canonPath(path); +} + + +Path canonPath(const Path & path, bool resolveSymlinks) +{ + string s; + + if (path[0] != '/') + throw Error(format("not an absolute path: `%1%'") % path); + + string::const_iterator i = path.begin(), end = path.end(); + string temp; + + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0, maxFollow = 1024; + + while (1) { + + /* Skip slashes. */ + while (i != end && *i == '/') i++; + if (i == end) break; + + /* Ignore `.'. */ + if (*i == '.' && (i + 1 == end || i[1] == '/')) + i++; + + /* If `..', delete the last component. */ + else if (*i == '.' && i + 1 < end && i[1] == '.' && + (i + 2 == end || i[2] == '/')) + { + if (!s.empty()) s.erase(s.rfind('/')); + i += 2; + } + + /* Normal component; copy it. */ + else { + s += '/'; + while (i != end && *i != '/') s += *i++; + + /* If s points to a symlink, resolve it and restart (since + the symlink target might contain new symlinks). */ + if (resolveSymlinks && isLink(s)) { + if (++followCount >= maxFollow) + throw Error(format("infinite symlink recursion in path `%1%'") % path); + temp = absPath(readLink(s), dirOf(s)) + + string(i, end); + i = temp.begin(); /* restart */ + end = temp.end(); + s = ""; + /* !!! potential for infinite loop */ + } + } + } + + return s.empty() ? "/" : s; +} + + +Path dirOf(const Path & path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == string::npos) + throw Error(format("invalid file name `%1%'") % path); + return pos == 0 ? "/" : Path(path, 0, pos); +} + + +string baseNameOf(const Path & path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == string::npos) + throw Error(format("invalid file name `%1%'") % path); + return string(path, pos + 1); +} + + +bool isInDir(const Path & path, const Path & dir) +{ + return path[0] == '/' + && string(path, 0, dir.size()) == dir + && path.size() >= dir.size() + 2 + && path[dir.size()] == '/'; +} + + +struct stat lstat(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of `%1%'") % path); + return st; +} + + +bool pathExists(const Path & path) +{ + int res; + struct stat st; + res = lstat(path.c_str(), &st); + if (!res) return true; + if (errno != ENOENT && errno != ENOTDIR) + throw SysError(format("getting status of %1%") % path); + return false; +} + + +Path readLink(const Path & path) +{ + checkInterrupt(); + struct stat st = lstat(path); + if (!S_ISLNK(st.st_mode)) + throw Error(format("`%1%' is not a symlink") % path); + char buf[st.st_size]; + if (readlink(path.c_str(), buf, st.st_size) != st.st_size) + throw SysError(format("reading symbolic link `%1%'") % path); + return string(buf, st.st_size); +} + + +bool isLink(const Path & path) +{ + struct stat st = lstat(path); + return S_ISLNK(st.st_mode); +} + + +Strings readDirectory(const Path & path) +{ + Strings names; + + AutoCloseDir dir = opendir(path.c_str()); + if (!dir) throw SysError(format("opening directory `%1%'") % path); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { /* sic */ + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + names.push_back(name); + } + if (errno) throw SysError(format("reading directory `%1%'") % path); + + return names; +} + + +string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + unsigned char * buf = new unsigned char[st.st_size]; + AutoDeleteArray<unsigned char> d(buf); + readFull(fd, buf, st.st_size); + + return string((char *) buf, st.st_size); +} + + +string readFile(const Path & path, bool drain) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY); + if (fd == -1) + throw SysError(format("opening file `%1%'") % path); + return drain ? drainFD(fd) : readFile(fd); +} + + +void writeFile(const Path & path, const string & s) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666); + if (fd == -1) + throw SysError(format("opening file `%1%'") % path); + writeFull(fd, (unsigned char *) s.data(), s.size()); +} + + +string readLine(int fd) +{ + string s; + while (1) { + checkInterrupt(); + char ch; + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void writeLine(int fd, string s) +{ + s += '\n'; + writeFull(fd, (const unsigned char *) s.data(), s.size()); +} + + +static void _deletePath(const Path & path, unsigned long long & bytesFreed) +{ + checkInterrupt(); + + printMsg(lvlVomit, format("%1%") % path); + + struct stat st = lstat(path); + + if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) + bytesFreed += st.st_blocks * 512; + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + + /* Make the directory writable. */ + if (!(st.st_mode & S_IWUSR)) { + if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) + throw SysError(format("making `%1%' writable") % path); + } + + for (Strings::iterator i = names.begin(); i != names.end(); ++i) + _deletePath(path + "/" + *i, bytesFreed); + } + + if (remove(path.c_str()) == -1) + throw SysError(format("cannot unlink `%1%'") % path); +} + + +void deletePath(const Path & path) +{ + unsigned long long dummy; + deletePath(path, dummy); +} + + +void deletePath(const Path & path, unsigned long long & bytesFreed) +{ + startNest(nest, lvlDebug, + format("recursively deleting path `%1%'") % path); + bytesFreed = 0; + _deletePath(path, bytesFreed); +} + + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + int & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR", "/tmp") : tmpRoot, true); + if (includePid) + return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); + else + return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); +} + + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static int globalCounter = 0; + int localCounter = 0; + int & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError(format("setting group of directory `%1%'") % tmpDir); + return tmpDir; + } + if (errno != EEXIST) + throw SysError(format("creating directory `%1%'") % tmpDir); + } +} + + +Paths createDirs(const Path & path) +{ + Paths created; + if (path == "/") return created; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + created = createDirs(dirOf(path)); + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) + throw SysError(format("creating directory `%1%'") % path); + st = lstat(path); + created.push_back(path); + } + + if (!S_ISDIR(st.st_mode)) throw Error(format("`%1%' is not a directory") % path); + + return created; +} + + +void createSymlink(const Path & target, const Path & link) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError(format("creating symlink from `%1%' to `%2%'") % link % target); +} + + +LogType logType = ltPretty; +Verbosity verbosity = lvlInfo; + +static int nestingLevel = 0; + + +Nest::Nest() +{ + nest = false; +} + + +Nest::~Nest() +{ + close(); +} + + +static string escVerbosity(Verbosity level) +{ + return int2String((int) level); +} + + +void Nest::open(Verbosity level, const FormatOrString & fs) +{ + if (level <= verbosity) { + if (logType == ltEscapes) + std::cerr << "\033[" << escVerbosity(level) << "p" + << fs.s << "\n"; + else + printMsg_(level, fs); + nest = true; + nestingLevel++; + } +} + + +void Nest::close() +{ + if (nest) { + nestingLevel--; + if (logType == ltEscapes) + std::cerr << "\033[q"; + nest = false; + } +} + + +void printMsg_(Verbosity level, const FormatOrString & fs) +{ + checkInterrupt(); + if (level > verbosity) return; + string prefix; + if (logType == ltPretty) + for (int i = 0; i < nestingLevel; i++) + prefix += "| "; + else if (logType == ltEscapes && level != lvlInfo) + prefix = "\033[" + escVerbosity(level) + "s"; + string s = (format("%1%%2%\n") % prefix % fs.s).str(); + writeToStderr(s); +} + + +void warnOnce(bool & haveWarned, const FormatOrString & fs) +{ + if (!haveWarned) { + printMsg(lvlError, format("warning: %1%") % fs.s); + haveWarned = true; + } +} + + +void writeToStderr(const string & s) +{ + try { + _writeToStderr((const unsigned char *) s.data(), s.size()); + } catch (SysError & e) { + /* Ignore failing writes to stderr if we're in an exception + handler, otherwise throw an exception. We need to ignore + write errors in exception handlers to ensure that cleanup + code runs to completion if the other side of stderr has + been closed unexpectedly. */ + if (!std::uncaught_exception()) throw; + } +} + + +static void defaultWriteToStderr(const unsigned char * buf, size_t count) +{ + writeFull(STDERR_FILENO, buf, count); +} + + +void (*_writeToStderr) (const unsigned char * buf, size_t count) = defaultWriteToStderr; + + +void readFull(int fd, unsigned char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, (char *) buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, const unsigned char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = write(fd, (char *) buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("writing to file"); + } + count -= res; + buf += res; + } +} + + +string drainFD(int fd) +{ + string result; + unsigned char buffer[4096]; + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buffer, sizeof buffer); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else result.append((char *) buffer, rd); + } + return result; +} + + + +////////////////////////////////////////////////////////////////////// + + +AutoDelete::AutoDelete(const string & p, bool recursive) : path(p) +{ + del = true; + this->recursive = recursive; +} + +AutoDelete::~AutoDelete() +{ + try { + if (del) { + if (recursive) + deletePath(path); + else { + if (remove(path.c_str()) == -1) + throw SysError(format("cannot unlink `%1%'") % path); + } + } + } catch (...) { + ignoreException(); + } +} + +void AutoDelete::cancel() +{ + del = false; +} + + + +////////////////////////////////////////////////////////////////////// + + +AutoCloseFD::AutoCloseFD() +{ + fd = -1; +} + + +AutoCloseFD::AutoCloseFD(int fd) +{ + this->fd = fd; +} + + +AutoCloseFD::AutoCloseFD(const AutoCloseFD & fd) +{ + /* Copying an AutoCloseFD isn't allowed (who should get to close + it?). But as an edge case, allow copying of closed + AutoCloseFDs. This is necessary due to tiresome reasons + involving copy constructor use on default object values in STL + containers (like when you do `map[value]' where value isn't in + the map yet). */ + this->fd = fd.fd; + if (this->fd != -1) abort(); +} + + +AutoCloseFD::~AutoCloseFD() +{ + try { + close(); + } catch (...) { + ignoreException(); + } +} + + +void AutoCloseFD::operator =(int fd) +{ + if (this->fd != fd) close(); + this->fd = fd; +} + + +AutoCloseFD::operator int() const +{ + return fd; +} + + +void AutoCloseFD::close() +{ + if (fd != -1) { + if (::close(fd) == -1) + /* This should never happen. */ + throw SysError(format("closing file descriptor %1%") % fd); + fd = -1; + } +} + + +bool AutoCloseFD::isOpen() +{ + return fd != -1; +} + + +/* Pass responsibility for closing this fd to the caller. */ +int AutoCloseFD::borrow() +{ + int oldFD = fd; + fd = -1; + return oldFD; +} + + +void Pipe::create() +{ + int fds[2]; + if (pipe(fds) != 0) throw SysError("creating pipe"); + readSide = fds[0]; + writeSide = fds[1]; + closeOnExec(readSide); + closeOnExec(writeSide); +} + + + +////////////////////////////////////////////////////////////////////// + + +AutoCloseDir::AutoCloseDir() +{ + dir = 0; +} + + +AutoCloseDir::AutoCloseDir(DIR * dir) +{ + this->dir = dir; +} + + +AutoCloseDir::~AutoCloseDir() +{ + close(); +} + + +void AutoCloseDir::operator =(DIR * dir) +{ + this->dir = dir; +} + + +AutoCloseDir::operator DIR *() +{ + return dir; +} + + +void AutoCloseDir::close() +{ + if (dir) { + closedir(dir); + dir = 0; + } +} + + +////////////////////////////////////////////////////////////////////// + + +Pid::Pid() +{ + pid = -1; + separatePG = false; + killSignal = SIGKILL; +} + + +Pid::~Pid() +{ + kill(); +} + + +void Pid::operator =(pid_t pid) +{ + if (this->pid != pid) kill(); + this->pid = pid; + killSignal = SIGKILL; // reset signal to default +} + + +Pid::operator pid_t() +{ + return pid; +} + + +void Pid::kill() +{ + if (pid == -1 || pid == 0) return; + + printMsg(lvlError, format("killing process %1%") % pid); + + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) + printMsg(lvlError, (SysError(format("killing process %1%") % pid).msg())); + + /* Wait until the child dies, disregarding the exit status. */ + int status; + while (waitpid(pid, &status, 0) == -1) { + checkInterrupt(); + if (errno != EINTR) { + printMsg(lvlError, + (SysError(format("waiting for process %1%") % pid).msg())); + break; + } + } + + pid = -1; +} + + +int Pid::wait(bool block) +{ + assert(pid != -1); + while (1) { + int status; + int res = waitpid(pid, &status, block ? 0 : WNOHANG); + if (res == pid) { + pid = -1; + return status; + } + if (res == 0 && !block) return -1; + if (errno != EINTR) + throw SysError("cannot get child exit status"); + checkInterrupt(); + } +} + + +void Pid::setSeparatePG(bool separatePG) +{ + this->separatePG = separatePG; +} + + +void Pid::setKillSignal(int signal) +{ + this->killSignal = signal; +} + + +void killUser(uid_t uid) +{ + debug(format("killing all processes running under uid `%1%'") % uid); + + assert(uid != 0); /* just to be safe... */ + + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ + + Pid pid; + pid = fork(); + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: + try { /* child */ + + if (setuid(uid) == -1) + throw SysError("setting uid"); + + while (true) { +#ifdef __APPLE__ + /* OSX's kill syscall takes a third parameter that, among other + things, determines if kill(-1, signo) affects the calling + process. In the OSX libc, it's set to true, which means + "follow POSIX", which we don't want here + */ + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; +#else + if (kill(-1, SIGKILL) == 0) break; +#endif + if (errno == ESRCH) break; /* no more processes */ + if (errno != EINTR) + throw SysError(format("cannot kill processes for uid `%1%'") % uid); + } + + } catch (std::exception & e) { + writeToStderr((format("killing processes belonging to uid `%1%': %2%\n") % uid % e.what()).str()); + _exit(1); + } + _exit(0); + } + + /* parent */ + int status = pid.wait(true); + if (status != 0) + throw Error(format("cannot kill processes for uid `%1%': %2%") % uid % statusToString(status)); + + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ +} + + +////////////////////////////////////////////////////////////////////// + + +string runProgram(Path program, bool searchPath, const Strings & args) +{ + checkInterrupt(); + + std::vector<const char *> cargs; /* careful with c_str()! */ + cargs.push_back(program.c_str()); + for (Strings::const_iterator i = args.begin(); i != args.end(); ++i) + cargs.push_back(i->c_str()); + cargs.push_back(0); + + /* Create a pipe. */ + Pipe pipe; + pipe.create(); + + /* Fork. */ + Pid pid; + pid = maybeVfork(); + + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: /* child */ + try { + if (dup2(pipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + + if (searchPath) + execvp(program.c_str(), (char * *) &cargs[0]); + else + execv(program.c_str(), (char * *) &cargs[0]); + throw SysError(format("executing `%1%'") % program); + + } catch (std::exception & e) { + writeToStderr("error: " + string(e.what()) + "\n"); + } + _exit(1); + } + + /* Parent. */ + + pipe.writeSide.close(); + + string result = drainFD(pipe.readSide); + + /* Wait for the child to finish. */ + int status = pid.wait(true); + if (!statusOk(status)) + throw Error(format("program `%1%' %2%") + % program % statusToString(status)); + + return result; +} + + +void closeMostFDs(const set<int> & exceptions) +{ + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO + && exceptions.find(fd) == exceptions.end()) + close(fd); /* ignore result */ +} + + +void closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + + +#if HAVE_VFORK +pid_t (*maybeVfork)() = vfork; +#else +pid_t (*maybeVfork)() = fork; +#endif + + +////////////////////////////////////////////////////////////////////// + + +volatile sig_atomic_t _isInterrupted = 0; + +void _interrupted() +{ + /* Block user interrupts while an exception is being handled. + Throwing an exception while another exception is being handled + kills the program! */ + if (!std::uncaught_exception()) { + _isInterrupted = 0; + throw Interrupted("interrupted by the user"); + } +} + + + +////////////////////////////////////////////////////////////////////// + + +template<class C> C tokenizeString(const string & s, const string & separators) +{ + C result; + string::size_type pos = s.find_first_not_of(separators, 0); + while (pos != string::npos) { + string::size_type end = s.find_first_of(separators, pos + 1); + if (end == string::npos) end = s.size(); + string token(s, pos, end - pos); + result.insert(result.end(), token); + pos = s.find_first_not_of(separators, end); + } + return result; +} + +template Strings tokenizeString(const string & s, const string & separators); +template StringSet tokenizeString(const string & s, const string & separators); +template vector<string> tokenizeString(const string & s, const string & separators); + + +string concatStringsSep(const string & sep, const Strings & ss) +{ + string s; + foreach (Strings::const_iterator, i, ss) { + if (s.size() != 0) s += sep; + s += *i; + } + return s; +} + + +string concatStringsSep(const string & sep, const StringSet & ss) +{ + string s; + foreach (StringSet::const_iterator, i, ss) { + if (s.size() != 0) s += sep; + s += *i; + } + return s; +} + + +string chomp(const string & s) +{ + size_t i = s.find_last_not_of(" \n\r\t"); + return i == string::npos ? "" : string(s, 0, i + 1); +} + + +string statusToString(int status) +{ + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) + return (format("failed with exit code %1%") % WEXITSTATUS(status)).str(); + else if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); +#if HAVE_STRSIGNAL + const char * description = strsignal(sig); + return (format("failed due to signal %1% (%2%)") % sig % description).str(); +#else + return (format("failed due to signal %1%") % sig).str(); +#endif + } + else + return "died abnormally"; + } else return "succeeded"; +} + + +bool statusOk(int status) +{ + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + + +bool hasSuffix(const string & s, const string & suffix) +{ + return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix; +} + + +void expect(std::istream & str, const string & s) +{ + char s2[s.size()]; + str.read(s2, s.size()); + if (string(s2, s.size()) != s) + throw Error(format("expected string `%1%'") % s); +} + + +string parseString(std::istream & str) +{ + string res; + expect(str, "\""); + int c; + while ((c = str.get()) != '"') + if (c == '\\') { + c = str.get(); + if (c == 'n') res += '\n'; + else if (c == 'r') res += '\r'; + else if (c == 't') res += '\t'; + else res += c; + } + else res += c; + return res; +} + + +bool endOfList(std::istream & str) +{ + if (str.peek() == ',') { + str.get(); + return false; + } + if (str.peek() == ']') { + str.get(); + return true; + } + return false; +} + + +string decodeOctalEscaped(const string & s) +{ + string r; + for (string::const_iterator i = s.begin(); i != s.end(); ) { + if (*i != '\\') { r += *i++; continue; } + unsigned char c = 0; + ++i; + while (i != s.end() && *i >= '0' && *i < '8') + c = c * 8 + (*i++ - '0'); + r += c; + } + return r; +} + + +void ignoreException() +{ + try { + throw; + } catch (std::exception & e) { + printMsg(lvlError, format("error (ignored): %1%") % e.what()); + } +} + + +} diff --git a/src/libutil/util.hh b/src/libutil/util.hh new file mode 100644 index 000000000000..8bedfea9a0de --- /dev/null +++ b/src/libutil/util.hh @@ -0,0 +1,349 @@ +#pragma once + +#include "types.hh" + +#include <sys/types.h> +#include <sys/stat.h> +#include <dirent.h> +#include <unistd.h> +#include <signal.h> + +#include <cstdio> + + +namespace nix { + + +#define foreach(it_type, it, collection) \ + for (it_type it = (collection).begin(); it != (collection).end(); ++it) + +#define foreach_reverse(it_type, it, collection) \ + for (it_type it = (collection).rbegin(); it != (collection).rend(); ++it) + + +/* Return an environment variable. */ +string getEnv(const string & key, const string & def = ""); + +/* Return an absolutized path, resolving paths relative to the + specified directory, or the current directory otherwise. The path + is also canonicalised. */ +Path absPath(Path path, Path dir = ""); + +/* Canonicalise a path by removing all `.' or `..' components and + double or trailing slashes. Optionally resolves all symlink + components such that each component of the resulting path is *not* + a symbolic link. */ +Path canonPath(const Path & path, bool resolveSymlinks = false); + +/* Return the directory part of the given canonical path, i.e., + everything before the final `/'. If the path is the root or an + immediate child thereof (e.g., `/foo'), this means an empty string + is returned. */ +Path dirOf(const Path & path); + +/* Return the base name of the given canonical path, i.e., everything + following the final `/'. */ +string baseNameOf(const Path & path); + +/* Check whether a given path is a descendant of the given + directory. */ +bool isInDir(const Path & path, const Path & dir); + +/* Get status of `path'. */ +struct stat lstat(const Path & path); + +/* Return true iff the given path exists. */ +bool pathExists(const Path & path); + +/* Read the contents (target) of a symbolic link. The result is not + in any way canonicalised. */ +Path readLink(const Path & path); + +bool isLink(const Path & path); + +/* Read the contents of a directory. The entries `.' and `..' are + removed. */ +Strings readDirectory(const Path & path); + +/* Read the contents of a file into a string. */ +string readFile(int fd); +string readFile(const Path & path, bool drain = false); + +/* Write a string to a file. */ +void writeFile(const Path & path, const string & s); + +/* Read a line from a file descriptor. */ +string readLine(int fd); + +/* Write a line to a file descriptor. */ +void writeLine(int fd, string s); + +/* Delete a path; i.e., in the case of a directory, it is deleted + recursively. Don't use this at home, kids. The second variant + returns the number of bytes and blocks freed. */ +void deletePath(const Path & path); + +void deletePath(const Path & path, unsigned long long & bytesFreed); + +/* Create a temporary directory. */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/* Create a directory and all its parents, if necessary. Returns the + list of created directories, in order of creation. */ +Paths createDirs(const Path & path); + +/* Create a symlink. */ +void createSymlink(const Path & target, const Path & link); + + +template<class T, class A> +T singleton(const A & a) +{ + T t; + t.insert(a); + return t; +} + + +/* Messages. */ + + +typedef enum { + ltPretty, /* nice, nested output */ + ltEscapes, /* nesting indicated using escape codes (for log2xml) */ + ltFlat /* no nesting */ +} LogType; + +extern LogType logType; +extern Verbosity verbosity; /* suppress msgs > this */ + +class Nest +{ +private: + bool nest; +public: + Nest(); + ~Nest(); + void open(Verbosity level, const FormatOrString & fs); + void close(); +}; + +void printMsg_(Verbosity level, const FormatOrString & fs); + +#define startNest(varName, level, f) \ + Nest varName; \ + if (level <= verbosity) { \ + varName.open(level, (f)); \ + } + +#define printMsg(level, f) \ + do { \ + if (level <= verbosity) { \ + printMsg_(level, (f)); \ + } \ + } while (0) + +#define debug(f) printMsg(lvlDebug, f) + +void warnOnce(bool & haveWarned, const FormatOrString & fs); + +void writeToStderr(const string & s); + +extern void (*_writeToStderr) (const unsigned char * buf, size_t count); + + +/* Wrappers arount read()/write() that read/write exactly the + requested number of bytes. */ +void readFull(int fd, unsigned char * buf, size_t count); +void writeFull(int fd, const unsigned char * buf, size_t count); + +MakeError(EndOfFile, Error) + + +/* Read a file descriptor until EOF occurs. */ +string drainFD(int fd); + + + +/* Automatic cleanup of resources. */ + + +template <class T> +struct AutoDeleteArray +{ + T * p; + AutoDeleteArray(T * p) : p(p) { } + ~AutoDeleteArray() + { + delete [] p; + } +}; + + +class AutoDelete +{ + Path path; + bool del; + bool recursive; +public: + AutoDelete(const Path & p, bool recursive = true); + ~AutoDelete(); + void cancel(); +}; + + +class AutoCloseFD +{ + int fd; +public: + AutoCloseFD(); + AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD & fd); + ~AutoCloseFD(); + void operator =(int fd); + operator int() const; + void close(); + bool isOpen(); + int borrow(); +}; + + +class Pipe +{ +public: + AutoCloseFD readSide, writeSide; + void create(); +}; + + +class AutoCloseDir +{ + DIR * dir; +public: + AutoCloseDir(); + AutoCloseDir(DIR * dir); + ~AutoCloseDir(); + void operator =(DIR * dir); + operator DIR *(); + void close(); +}; + + +class Pid +{ + pid_t pid; + bool separatePG; + int killSignal; +public: + Pid(); + ~Pid(); + void operator =(pid_t pid); + operator pid_t(); + void kill(); + int wait(bool block); + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); +}; + + +/* Kill all processes running under the specified uid by sending them + a SIGKILL. */ +void killUser(uid_t uid); + + +/* Run a program and return its stdout in a string (i.e., like the + shell backtick operator). */ +string runProgram(Path program, bool searchPath = false, + const Strings & args = Strings()); + +/* Close all file descriptors except stdin, stdout, stderr, and those + listed in the given set. Good practice in child processes. */ +void closeMostFDs(const set<int> & exceptions); + +/* Set the close-on-exec flag for the given file descriptor. */ +void closeOnExec(int fd); + +/* Call vfork() if available, otherwise fork(). */ +extern pid_t (*maybeVfork)(); + + +/* User interruption. */ + +extern volatile sig_atomic_t _isInterrupted; + +void _interrupted(); + +void inline checkInterrupt() +{ + if (_isInterrupted) _interrupted(); +} + +MakeError(Interrupted, BaseError) + + +/* String tokenizer. */ +template<class C> C tokenizeString(const string & s, const string & separators = " \t\n\r"); + + +/* Concatenate the given strings with a separator between the + elements. */ +string concatStringsSep(const string & sep, const Strings & ss); +string concatStringsSep(const string & sep, const StringSet & ss); + + +/* Remove trailing whitespace from a string. */ +string chomp(const string & s); + + +/* Convert the exit status of a child as returned by wait() into an + error string. */ +string statusToString(int status); + +bool statusOk(int status); + + +/* Parse a string into an integer. */ +template<class N> bool string2Int(const string & s, N & n) +{ + std::istringstream str(s); + str >> n; + return str && str.get() == EOF; +} + +template<class N> string int2String(N n) +{ + std::ostringstream str; + str << n; + return str.str(); +} + + +/* Return true iff `s' ends in `suffix'. */ +bool hasSuffix(const string & s, const string & suffix); + + +/* Read string `s' from stream `str'. */ +void expect(std::istream & str, const string & s); + + +/* Read a C-style string from stream `str'. */ +string parseString(std::istream & str); + + +/* Utility function used to parse legacy ATerms. */ +bool endOfList(std::istream & str); + + +/* Escape a string that contains octal-encoded escape codes such as + used in /etc/fstab and /proc/mounts (e.g. "foo\040bar" decodes to + "foo bar"). */ +string decodeOctalEscaped(const string & s); + + +/* Exception handling in destructors: print an error message, then + ignore the exception. */ +void ignoreException(); + + +} diff --git a/src/libutil/xml-writer.cc b/src/libutil/xml-writer.cc new file mode 100644 index 000000000000..01794001b2c6 --- /dev/null +++ b/src/libutil/xml-writer.cc @@ -0,0 +1,94 @@ +#include <assert.h> + +#include "xml-writer.hh" + + +namespace nix { + + +XMLWriter::XMLWriter(bool indent, std::ostream & output) + : output(output), indent(indent) +{ + output << "<?xml version='1.0' encoding='utf-8'?>" << std::endl; + closed = false; +} + + +XMLWriter::~XMLWriter() +{ + close(); +} + + +void XMLWriter::close() +{ + if (closed) return; + while (!pendingElems.empty()) closeElement(); + closed = true; +} + + +void XMLWriter::indent_(unsigned int depth) +{ + if (!indent) return; + output << string(depth * 2, ' '); +} + + +void XMLWriter::openElement(const string & name, + const XMLAttrs & attrs) +{ + assert(!closed); + indent_(pendingElems.size()); + output << "<" << name; + writeAttrs(attrs); + output << ">"; + if (indent) output << std::endl; + pendingElems.push_back(name); +} + + +void XMLWriter::closeElement() +{ + assert(!pendingElems.empty()); + indent_(pendingElems.size() - 1); + output << "</" << pendingElems.back() << ">"; + if (indent) output << std::endl; + pendingElems.pop_back(); + if (pendingElems.empty()) closed = true; +} + + +void XMLWriter::writeEmptyElement(const string & name, + const XMLAttrs & attrs) +{ + assert(!closed); + indent_(pendingElems.size()); + output << "<" << name; + writeAttrs(attrs); + output << " />"; + if (indent) output << std::endl; +} + + +void XMLWriter::writeAttrs(const XMLAttrs & attrs) +{ + for (XMLAttrs::const_iterator i = attrs.begin(); i != attrs.end(); ++i) { + output << " " << i->first << "=\""; + for (unsigned int j = 0; j < i->second.size(); ++j) { + char c = i->second[j]; + if (c == '"') output << """; + else if (c == '<') output << "<"; + else if (c == '>') output << ">"; + else if (c == '&') output << "&"; + /* Escape newlines to prevent attribute normalisation (see + XML spec, section 3.3.3. */ + else if (c == '\n') output << "
"; + else output << c; + } + output << "\""; + } +} + + +} diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh new file mode 100644 index 000000000000..3cefe3712c08 --- /dev/null +++ b/src/libutil/xml-writer.hh @@ -0,0 +1,69 @@ +#pragma once + +#include <iostream> +#include <string> +#include <list> +#include <map> + + +namespace nix { + +using std::string; +using std::map; +using std::list; + + +typedef map<string, string> XMLAttrs; + + +class XMLWriter +{ +private: + + std::ostream & output; + + bool indent; + bool closed; + + list<string> pendingElems; + +public: + + XMLWriter(bool indent, std::ostream & output); + ~XMLWriter(); + + void close(); + + void openElement(const string & name, + const XMLAttrs & attrs = XMLAttrs()); + void closeElement(); + + void writeEmptyElement(const string & name, + const XMLAttrs & attrs = XMLAttrs()); + +private: + void writeAttrs(const XMLAttrs & attrs); + + void indent_(unsigned int depth); +}; + + +class XMLOpenElement +{ +private: + XMLWriter & writer; +public: + XMLOpenElement(XMLWriter & writer, const string & name, + const XMLAttrs & attrs = XMLAttrs()) + : writer(writer) + { + writer.openElement(name, attrs); + } + ~XMLOpenElement() + { + writer.closeElement(); + } +}; + + +} |