#include "serialise.hh" #include <boost/coroutine2/coroutine.hpp> #include <cerrno> #include <cstring> #include <memory> #include "glog/logging.h" #include "util.hh" namespace nix { void BufferedSink::operator()(const unsigned char* data, size_t len) { if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); while (len) { /* Optimisation: bypass the buffer if the data exceeds the buffer size. */ if (bufPos + len >= bufSize) { flush(); write(data, len); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer when it's full. */ size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; memcpy(buffer.get() + bufPos, data, n); data += n; bufPos += n; len -= n; if (bufPos == bufSize) flush(); } } void BufferedSink::flush() { if (bufPos == 0) return; size_t n = bufPos; bufPos = 0; // don't trigger the assert() in ~BufferedSink() write(buffer.get(), n); } FdSink::~FdSink() { try { flush(); } catch (...) { ignoreException(); } } size_t threshold = 256 * 1024 * 1024; static void warnLargeDump() { LOG(WARNING) << "dumping very large path (> 256 MiB); this may run out of memory"; } void FdSink::write(const unsigned char* data, size_t len) { written += len; static bool warned = false; if (warn && !warned) { if (written > threshold) { warnLargeDump(); warned = true; } } try { writeFull(fd, data, len); } catch (SysError& e) { _good = false; throw; } } bool FdSink::good() { return _good; } void Source::operator()(unsigned char* data, size_t len) { while (len) { size_t n = read(data, len); data += n; len -= n; } } std::string Source::drain() { std::string s; std::vector<unsigned char> buf(8192); while (true) { size_t n; try { n = read(buf.data(), buf.size()); s.append((char*)buf.data(), n); } catch (EndOfFile&) { break; } } return s; } size_t BufferedSource::read(unsigned char* data, size_t len) { if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize); /* Copy out the data in the buffer. */ size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; memcpy(data, buffer.get() + bufPosOut, n); bufPosOut += n; if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; return n; } bool BufferedSource::hasData() { return bufPosOut < bufPosIn; } size_t FdSource::readUnbuffered(unsigned char* data, size_t len) { ssize_t n; do { checkInterrupt(); n = ::read(fd, (char*)data, len); } while (n == -1 && errno == EINTR); if (n == -1) { _good = false; throw SysError("reading from file"); } if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } read += n; return n; } bool FdSource::good() { return _good; } size_t StringSource::read(unsigned char* data, size_t len) { if (pos == s.size()) throw EndOfFile("end of string reached"); size_t n = s.copy((char*)data, len, pos); pos += n; return n; } #if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600 #error Coroutines are broken in this version of Boost! #endif std::unique_ptr<Source> sinkToSource(std::function<void(Sink&)> fun, std::function<void()> eof) { struct SinkToSource : Source { typedef boost::coroutines2::coroutine<std::string> coro_t; std::function<void(Sink&)> fun; std::function<void()> eof; std::optional<coro_t::pull_type> coro; bool started = false; SinkToSource(std::function<void(Sink&)> fun, std::function<void()> eof) : fun(fun), eof(eof) {} std::string cur; size_t pos = 0; size_t read(unsigned char* data, size_t len) override { if (!coro) coro = coro_t::pull_type([&](coro_t::push_type& yield) { LambdaSink sink([&](const unsigned char* data, size_t len) { if (len) yield(std::string((const char*)data, len)); }); fun(sink); }); if (!*coro) { eof(); abort(); } if (pos == cur.size()) { if (!cur.empty()) (*coro)(); cur = coro->get(); pos = 0; } auto n = std::min(cur.size() - pos, len); memcpy(data, (unsigned char*)cur.data() + pos, n); pos += n; return n; } }; return std::make_unique<SinkToSource>(fun, eof); } void writePadding(size_t len, Sink& sink) { if (len % 8) { unsigned char zero[8]; memset(zero, 0, sizeof(zero)); sink(zero, 8 - (len % 8)); } } void writeString(const unsigned char* buf, size_t len, Sink& sink) { sink << len; sink(buf, len); writePadding(len, sink); } Sink& operator<<(Sink& sink, const string& s) { writeString((const unsigned char*)s.data(), s.size(), sink); return sink; } template <class T> void writeStrings(const T& ss, Sink& sink) { sink << ss.size(); for (auto& i : ss) { sink << i; } } Sink& operator<<(Sink& sink, const Strings& s) { writeStrings(s, sink); return sink; } Sink& operator<<(Sink& sink, const StringSet& s) { writeStrings(s, sink); return sink; } void readPadding(size_t len, Source& source) { if (len % 8) { unsigned char zero[8]; size_t n = 8 - (len % 8); source(zero, n); for (unsigned int i = 0; i < n; i++) if (zero[i]) throw SerialisationError("non-zero padding"); } } size_t readString(unsigned char* buf, size_t max, Source& source) { auto len = readNum<size_t>(source); if (len > max) throw SerialisationError("string is too long"); source(buf, len); readPadding(len, source); return len; } string readString(Source& source, size_t max) { auto len = readNum<size_t>(source); if (len > max) throw SerialisationError("string is too long"); std::string res(len, 0); source((unsigned char*)res.data(), len); readPadding(len, source); return res; } Source& operator>>(Source& in, string& s) { s = readString(in); return in; } template <class T> T readStrings(Source& source) { auto count = readNum<size_t>(source); T ss; while (count--) ss.insert(ss.end(), readString(source)); return ss; } template Paths readStrings(Source& source); template PathSet readStrings(Source& source); void StringSink::operator()(const unsigned char* data, size_t len) { static bool warned = false; if (!warned && s->size() > threshold) { warnLargeDump(); warned = true; } s->append((const char*)data, len); } } // namespace nix