feat(3p/nix): Add main function for grpc daemon r/1472

Implement the main function for the new, proto-based nix daemon. This
mostly replicates the behavior of the previous nix daemon - it starts a
grpc server listening on the same nix socket path and supports the same
set of options - with the exception of --stdio, which has been renamed
to --pipe and documented in the man page.

Change-Id: Ib729283c1d5d35c0e1b0a968bc1f052f5527f2d7
Reviewed-on: https://cl.tvl.fyi/c/depot/+/1356
Tested-by: BuildkiteCI
Reviewed-by: tazjin <mail@tazj.in>
Reviewed-by: kanepyork <rikingcoding@gmail.com>

10 files changed, 174 insertions, 14 deletions

diff --git a/third_party/nix/doc/manual/command-ref/nix-daemon.xml b/third_party/nix/doc/manual/command-ref/nix-daemon.xml
index a2161f0334..9159d15d1c 100644
--- a/third_party/nix/doc/manual/command-ref/nix-daemon.xml
+++ b/third_party/nix/doc/manual/command-ref/nix-daemon.xml
@@ -32,4 +32,20 @@ of unprivileged users.</para>
 
 </refsection>
 
+<refsection><title>Options</title>
+
+  <variablelist>
+
+  <varlistentry><term><option>--pipe</option></term>
+
+    <listitem><para>Causes the nix daemon to forward stdin and stdout to and
+    from the actual daemon socket. This is used when communicating with a remote
+    store over SSH</para></listitem>
+
+  </varlistentry>
+
+  </variablelist>
+
+</refsection>
+
 </refentry>
diff --git a/third_party/nix/misc/systemd/nix-daemon.service.in b/third_party/nix/misc/systemd/nix-daemon.service.in
index 25655204d4..c3d2a4a39e 100644
--- a/third_party/nix/misc/systemd/nix-daemon.service.in
+++ b/third_party/nix/misc/systemd/nix-daemon.service.in
@@ -5,7 +5,7 @@ RequiresMountsFor=@localstatedir@
 ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
 
 [Service]
-ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
+ExecStart=@@bindir@/nix-daemon nix-daemon
 KillMode=process
 
 [Install]
diff --git a/third_party/nix/src/CMakeLists.txt b/third_party/nix/src/CMakeLists.txt
index 3775333ee3..a142bfafb4 100644
--- a/third_party/nix/src/CMakeLists.txt
+++ b/third_party/nix/src/CMakeLists.txt
@@ -61,8 +61,8 @@ target_sources(nix
     nix-channel/nix-channel.cc
     nix-collect-garbage/nix-collect-garbage.cc
     nix-copy-closure/nix-copy-closure.cc
-    nix-daemon/nix-daemon.cc
     nix-daemon/nix-daemon-proto.cc
+    nix-daemon/nix-daemon-main.cc
     nix-env/nix-env.cc
     nix-env/user-env.cc
     nix-instantiate/nix-instantiate.cc
diff --git a/third_party/nix/src/libstore/derivations.cc b/third_party/nix/src/libstore/derivations.cc
index 419fc584f1..18b313385c 100644
--- a/third_party/nix/src/libstore/derivations.cc
+++ b/third_party/nix/src/libstore/derivations.cc
@@ -31,11 +31,11 @@ void DerivationOutput::parseHashInfo(bool& recursive, Hash& hash) const {
 }
 
 BasicDerivation BasicDerivation::from_proto(
-    const nix::proto::Derivation* proto_derivation, const nix::Store* store) {
+    const nix::proto::Derivation* proto_derivation, const nix::Store& store) {
   BasicDerivation result;
   result.platform = proto_derivation->platform();
   result.builder = proto_derivation->builder().path();
-  store->assertStorePath(result.builder);
+  store.assertStorePath(result.builder);
 
   result.outputs.insert(proto_derivation->outputs().begin(),
                         proto_derivation->outputs().end());
diff --git a/third_party/nix/src/libstore/derivations.hh b/third_party/nix/src/libstore/derivations.hh
index d8a5dbf092..21ef71372e 100644
--- a/third_party/nix/src/libstore/derivations.hh
+++ b/third_party/nix/src/libstore/derivations.hh
@@ -59,7 +59,7 @@ struct BasicDerivation {
   // Convert the given proto derivation to a BasicDerivation in the given
   // nix::Store.
   static BasicDerivation from_proto(
-      const nix::proto::Derivation* proto_derivation, const nix::Store* store);
+      const nix::proto::Derivation* proto_derivation, const nix::Store& store);
 
   virtual ~BasicDerivation(){};
 
diff --git a/third_party/nix/src/libstore/ssh-store.cc b/third_party/nix/src/libstore/ssh-store.cc
index eae8ee888b..48fea858a3 100644
--- a/third_party/nix/src/libstore/ssh-store.cc
+++ b/third_party/nix/src/libstore/ssh-store.cc
@@ -67,7 +67,7 @@ ref<FSAccessor> SSHStore::getFSAccessor() {
 
 ref<RemoteStore::Connection> SSHStore::openConnection() {
   auto conn = make_ref<Connection>();
-  conn->sshConn = master.startCommand("nix-daemon --stdio");
+  conn->sshConn = master.startCommand("nix-daemon --pipe");
   conn->to = FdSink(conn->sshConn->in.get());
   conn->from = FdSource(conn->sshConn->out.get());
   initConnection(*conn);
diff --git a/third_party/nix/src/libutil/util.cc b/third_party/nix/src/libutil/util.cc
index f69c341c2c..75a8e73eac 100644
--- a/third_party/nix/src/libutil/util.cc
+++ b/third_party/nix/src/libutil/util.cc
@@ -167,6 +167,7 @@ Path canonPath(const Path& path, bool resolveSymlinks) {
   return s.empty() ? "/" : s;
 }
 
+// TODO(grfn) remove in favor of std::filesystem::path::parent_path()
 Path dirOf(absl::string_view path) {
   Path::size_type pos = path.rfind('/');
   if (pos == std::string::npos) {
@@ -175,6 +176,7 @@ Path dirOf(absl::string_view path) {
   return pos == 0 ? "/" : Path(path, 0, pos);
 }
 
+// TODO(grfn) remove in favor of std::filesystem::path::root_name()
 std::string baseNameOf(const Path& path) {
   if (path.empty()) {
     return "";
@@ -558,6 +560,7 @@ Path getDataDir() {
   return dataDir;
 }
 
+// TODO(grfn): Remove in favor of std::filesystem::create_directories
 Paths createDirs(const Path& path) {
   Paths created;
   if (path == "/") {
diff --git a/third_party/nix/src/nix-daemon/nix-daemon-main.cc b/third_party/nix/src/nix-daemon/nix-daemon-main.cc
new file mode 100644
index 0000000000..e492c343b9
--- /dev/null
+++ b/third_party/nix/src/nix-daemon/nix-daemon-main.cc
@@ -0,0 +1,143 @@
+#include <filesystem>
+
+#include <absl/strings/str_format.h>
+#include <fcntl.h>
+#include <glog/logging.h>
+#include <grpcpp/security/server_credentials.h>
+#include <grpcpp/server.h>
+#include <grpcpp/server_builder_impl.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#include "libmain/shared.hh"
+#include "libstore/globals.hh"
+#include "libstore/store-api.hh"
+#include "libutil/util.hh"
+#include "nix-daemon-proto.hh"
+#include "nix-daemon/nix-daemon-proto.hh"
+#include "nix/legacy.hh"
+
+namespace nix::daemon {
+
+using grpc::Server;
+using grpc_impl::ServerBuilder;
+
+// TODO(grfn): There has to be a better way to do this - this was ported
+// verbatim from the old daemon implementation without much critical evaluation.
+static int ForwardToSocket(nix::Path socket_path) {
+  // Forward on this connection to the real daemon
+  int sockfd = socket(PF_UNIX, SOCK_STREAM, 0);
+  if (sockfd == -1) {
+    throw SysError("creating Unix domain socket");
+  }
+
+  auto socketDir = dirOf(socket_path);
+  if (chdir(socketDir.c_str()) == -1) {
+    throw SysError(format("changing to socket directory '%1%'") % socketDir);
+  }
+
+  auto socketName = baseNameOf(socket_path);
+  auto addr = sockaddr_un{};
+  addr.sun_family = AF_UNIX;
+  if (socketName.size() + 1 >= sizeof(addr.sun_path)) {
+    throw Error(format("socket name %1% is too long") % socketName);
+  }
+  strncpy(addr.sun_path, socketName.c_str(), sizeof(addr.sun_family));
+
+  if (connect(sockfd, reinterpret_cast<struct sockaddr*>(&addr),
+              sizeof(addr)) == -1) {
+    throw SysError(format("cannot connect to daemon at %1%") % socket_path);
+  }
+
+  auto nfds = (sockfd > STDIN_FILENO ? sockfd : STDIN_FILENO) + 1;
+  while (true) {
+    fd_set fds;
+    FD_ZERO(&fds);
+    FD_SET(sockfd, &fds);
+    FD_SET(STDIN_FILENO, &fds);
+    if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) {
+      throw SysError("waiting for data from client or server");
+    }
+    if (FD_ISSET(sockfd, &fds)) {
+      auto res = splice(sockfd, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX,
+                        SPLICE_F_MOVE);
+      if (res == -1) {
+        throw SysError("splicing data from daemon socket to stdout");
+      }
+      if (res == 0) {
+        throw EndOfFile("unexpected EOF from daemon socket");
+      }
+    }
+    if (FD_ISSET(STDIN_FILENO, &fds)) {
+      auto res = splice(STDIN_FILENO, nullptr, sockfd, nullptr, SSIZE_MAX,
+                        SPLICE_F_MOVE);
+      if (res == -1) {
+        throw SysError("splicing data from stdin to daemon socket");
+      }
+      if (res == 0) {
+        return 0;
+      }
+    }
+  }
+}
+
+int RunServer() {
+  Store::Params params;
+  params["path-info-cache-size"] = "0";
+  auto store = openStore(settings.storeUri, params);
+  auto worker = NewWorkerService(*store);
+
+  std::filesystem::path socket_path(settings.nixDaemonSocketFile);
+  std::filesystem::create_directories(socket_path.parent_path());
+  auto socket_addr = absl::StrFormat("unix://%s", socket_path);
+
+  ServerBuilder builder;
+  builder.AddListeningPort(socket_addr, grpc::InsecureServerCredentials());
+  builder.RegisterService(worker);
+
+  std::unique_ptr<Server> server(builder.BuildAndStart());
+  if (server) {
+    LOG(INFO) << "Nix daemon listening at " << socket_addr;
+    server->Wait();
+    return 0;
+  } else {
+    return 1;
+  }
+}
+
+static int main_(int argc, char** argv) {
+  auto pipe = false;
+
+  // TODO(grfn): Replace with absl::flags
+  parseCmdLine(argc, argv,
+               [&](Strings::iterator& arg, const Strings::iterator& end) {
+                 if (*arg == "--help") {
+                   showManPage("nix-daemon");
+                 } else if (*arg == "--version") {
+                   printVersion("nix-daemon");
+                 } else if (*arg == "--pipe") {
+                   // Causes the daemon to forward stdin and stdout to and from
+                   // the actual daemon socket
+                   pipe = true;
+                 } else {
+                   return false;
+                 }
+                 return true;
+               });
+
+  if (pipe) {
+    if (getStoreType() == tDaemon) {
+      return ForwardToSocket(settings.nixDaemonSocketFile);
+    } else {
+      // TODO(grfn): Need to launch a server on stdin here - upstream calls
+      // processConnection(true, "root", 0);
+      throw "Not implemented";
+    }
+  }
+  return RunServer();
+}
+
+// TODO(grfn): Replace this with something less magical
+static RegisterLegacyCommand s1("nix-daemon", main_);
+
+}  // namespace nix::daemon
diff --git a/third_party/nix/src/nix-daemon/nix-daemon-proto.cc b/third_party/nix/src/nix-daemon/nix-daemon-proto.cc
index ffd34cc60a..d382964a71 100644
--- a/third_party/nix/src/nix-daemon/nix-daemon-proto.cc
+++ b/third_party/nix/src/nix-daemon/nix-daemon-proto.cc
@@ -26,7 +26,7 @@ static Status INVALID_STORE_PATH =
 
 class WorkerServiceImpl final : public WorkerService::Service {
  public:
-  WorkerServiceImpl(nix::Store* store) : store_(store) {}
+  WorkerServiceImpl(nix::Store& store) : store_(&store) {}
 
   Status IsValidPath(grpc::ServerContext* context, const StorePath* request,
                      nix::proto::IsValidPathResponse* response) override {
@@ -221,7 +221,7 @@ class WorkerServiceImpl final : public WorkerService::Service {
       nix::proto::BuildDerivationResponse* response) override {
     auto drv_path = request->drv_path().path();
     store_->assertStorePath(drv_path);
-    auto drv = BasicDerivation::from_proto(&request->derivation(), store_);
+    auto drv = BasicDerivation::from_proto(&request->derivation(), *store_);
 
     auto build_mode = nix::build_mode_from(request->build_mode());
     if (!build_mode) {
@@ -282,12 +282,11 @@ class WorkerServiceImpl final : public WorkerService::Service {
   };
 
  private:
-  // TODO(tazjin): Who owns the store?
-  nix::Store* store_;
+  ref<nix::Store> store_;
 };
 
-std::unique_ptr<WorkerService::Service> NewWorkerService(nix::Store* store) {
-  return std::make_unique<WorkerServiceImpl>(store);
+WorkerService::Service* NewWorkerService(nix::Store& store) {
+  return new WorkerServiceImpl(store);
 }
 
 }  // namespace nix::daemon
diff --git a/third_party/nix/src/nix-daemon/nix-daemon-proto.hh b/third_party/nix/src/nix-daemon/nix-daemon-proto.hh
index 5ffab9ebbe..ca871213eb 100644
--- a/third_party/nix/src/nix-daemon/nix-daemon-proto.hh
+++ b/third_party/nix/src/nix-daemon/nix-daemon-proto.hh
@@ -7,7 +7,6 @@
 
 namespace nix::daemon {
 
-std::unique_ptr<nix::proto::WorkerService::Service> NewWorkerService(
-    nix::Store*);
+nix::proto::WorkerService::Service* NewWorkerService(nix::Store&);
 
 }  // namespace nix::daemon