refactor: Move nixpkgs attribute to `third_party.nixpkgs` r/2470

Please read b/108 to make sense of this.

This gets rid of the explicit list of exposed packages from nixpkgs,
and instead makes the entire package set available at
`third_party.nixpkgs`.

To accommodate this, a LOT of things have to be very slightly shuffled
around. Some of this was done in already submitted CLs, but this
change is unfortunately still quite noisy.

Pay extra attention to:

* overlay-like functionality that was partially moved to actual
  overlays (partially as in, the minimum required to get a green
  build)

* modified uses of the package set path, esp. in NixOS systems

Special notes:

* xanthous has been disabled in CI because of issues with the Haskell
  overlay
* //third_party/nix has been disabled because of other unclear
  dependency issues

Both of these will be tackled in a followup CL.

Change-Id: I2f9c60a4d275fdb5209264be0addfd7e06c53118
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2910
Reviewed-by: glittershark <grfn@gws.fyi>
Reviewed-by: sterni <sternenseemann@systemli.org>
Tested-by: BuildkiteCI

6 files changed, 74 insertions, 233 deletions

diff --git a/third_party/nix/default.nix b/third_party/nix/default.nix
index b48551cff0d1..d3adabe2bdd8 100644
--- a/third_party/nix/default.nix
+++ b/third_party/nix/default.nix
@@ -1,5 +1,6 @@
 args@{
-  pkgs ? (import ../.. {}).third_party
+  depot ? (import ../.. {})
+, pkgs ? depot.third_party.nixpkgs
 , lib
 , buildType ? "release"
 , depotPath ? ../..
@@ -43,10 +44,11 @@ let
   protoSrcs = pkgs.runCommand "nix-proto-srcs" {} ''
     export PROTO_SRCS=${./src/proto}
     mkdir -p $out/libproto
-    ${pkgs.protobuf}/bin/protoc -I=$PROTO_SRCS \
+    ${depot.third_party.protobuf}/bin/protoc -I=$PROTO_SRCS \
       --cpp_out=$out/libproto \
-      --plugin=protoc-gen-grpc=${pkgs.grpc}/bin/grpc_cpp_plugin --grpc_out=$out/libproto \
-      $PROTO_SRCS/*.proto
+      --plugin=protoc-gen-grpc=${depot.third_party.grpc}/bin/grpc_cpp_plugin \
+        --grpc_out=$out/libproto \
+        $PROTO_SRCS/*.proto
   '';
 
   # Derivation for busybox that just has the `busybox` binary in bin/, not all
@@ -73,7 +75,6 @@ in lib.fix (self: pkgs.llvmPackages.libcxxStdenv.mkDerivation {
 
  # TODO(tazjin): Some of these might only be required for native inputs
   buildInputs = with pkgs; [
-    abseil_cpp
     aws-s3-cpp
     brotli
     bzip2
@@ -81,16 +82,18 @@ in lib.fix (self: pkgs.llvmPackages.libcxxStdenv.mkDerivation {
     curl
     editline
     flex
-    glog
-    grpc
     libseccomp
     libsodium
     systemd.dev
     openssl
-    protobuf
     sqlite
     xz
-  ];
+  ] ++ (with depot.third_party; [
+    abseil_cpp
+    glog
+    grpc
+    protobuf
+  ]);
 
   doCheck = false;
   doInstallCheck = true;
@@ -99,8 +102,8 @@ in lib.fix (self: pkgs.llvmPackages.libcxxStdenv.mkDerivation {
   dontStrip = true;
 
   installCheckInputs = with pkgs; [
+    depot.third_party.gtest
     fd
-    gtest
     rapidcheck
   ];
 
@@ -185,6 +188,9 @@ in lib.fix (self: pkgs.llvmPackages.libcxxStdenv.mkDerivation {
   # TODO(tazjin): integration test setup?
   # TODO(tazjin): docs generation?
 
+  # TODO(tazjin): Sort out after CL/2910 lands
+  meta.ci = false;
+
   passthru = {
     build-shell = self.overrideAttrs (up: rec {
       run_clang_tidy = pkgs.writeShellScriptBin "run-clang-tidy" ''
diff --git a/third_party/nix/test-vm.nix b/third_party/nix/test-vm.nix
index 8f77441b7dbd..550537679c49 100644
--- a/third_party/nix/test-vm.nix
+++ b/third_party/nix/test-vm.nix
@@ -1,10 +1,9 @@
 { depot, pkgs, ... }:
 
 let
-
   configuration = { ... }: {
     imports = [
-      "${pkgs.nixpkgsSrc}/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix"
+      "${pkgs.path}/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix"
     ];
 
     nix.package = depot.third_party.nix;
@@ -16,6 +15,5 @@ let
     ];
   };
 
-  system = pkgs.nixos { inherit configuration; };
-
+  system = depot.third_party.nixos { inherit configuration; };
 in system.vm
diff --git a/third_party/nixpkgs-exposed/.skip-subtree b/third_party/nixpkgs-exposed/.skip-subtree
deleted file mode 100644
index aff471145e46..000000000000
--- a/third_party/nixpkgs-exposed/.skip-subtree
+++ /dev/null
@@ -1 +0,0 @@
-This subtree is responsible for setting up the read tree arguments in the first place.
diff --git a/third_party/nixpkgs-exposed/OWNERS b/third_party/nixpkgs-exposed/OWNERS
deleted file mode 100644
index 72c0ac47fb47..000000000000
--- a/third_party/nixpkgs-exposed/OWNERS
+++ /dev/null
@@ -1,8 +0,0 @@
-# The nixpkgs whitelist needs to be changed every time we reference
-# a new attribute, so every commiter should be able to change it,
-# otherwise we create undue blockers on superowners.
-inherited: true
-owners:
-  - Profpatsch
-  - sterni
-  - glittershark
diff --git a/third_party/nixpkgs-exposed/exposed/default.nix b/third_party/nixpkgs-exposed/exposed/default.nix
deleted file mode 100644
index 478b55cb2965..000000000000
--- a/third_party/nixpkgs-exposed/exposed/default.nix
+++ /dev/null
@@ -1,210 +0,0 @@
-# This file has to be in yet another subdir
-# because of how readTree interprets .skip-subtree
-# see https://b.tvl.fyi/issues/89
-{ nixpkgs, stableNixpkgs }:
-{
-  # Inherit the packages from nixos-unstable that should be available inside
-  # of the repo. They become available under `pkgs.third_party.<name>`
-  inherit (nixpkgs)
-    age
-    atk
-    autoconf
-    autoreconfHook
-    avrdude
-    avrlibc
-    bashInteractive
-    bat
-    bc
-    bind
-    buildBazelPackage
-    buildFHSUserEnv
-    buildGoModule
-    buildGoPackage
-    buildPackages
-    buildRustCrate
-    buildkite-agent
-    busybox
-    bzip2
-    c-ares
-    cacert
-    cachix
-    cairo
-    cargo
-    cgit
-    clang_11
-    cmake
-    coreutils
-    cudatoolkit
-    darwin
-    dbus-glib
-    dbus_libs
-    dfu-programmer
-    dfu-util
-    diffutils
-    docker-compose
-    dockerTools
-    emacs27
-    emacs27-nox
-    emacsPackages
-    emacsPackagesGen
-    execline
-    fd
-    fdtools
-    fetchFromGitHub
-    fetchFromGitLab
-    fetchgit
-    fetchurl
-    fetchzip
-    ffmpeg-full
-    findutils
-    fira
-    fira-code
-    fira-mono
-    flamegraph
-    fontconfig
-    freetype
-    gdk-pixbuf
-    gettext
-    glibc
-    glib
-    gtk3
-    gmock
-    gnused
-    gnutar
-    go
-    google-cloud-sdk
-    graphviz
-    gzip
-    haskell
-    iana-etc
-    imagemagickBig
-    installShellFiles
-    jdk
-    jdk11
-    jdk11_headless
-    jetbrains-mono
-    jq
-    kontemplate
-    lib
-    libappindicator-gtk3
-    libredirect
-    libffi
-    linuxPackages
-    linuxPackages_5_11
-    lr
-    luajit
-    lutris
-    makeFontsConf
-    makeWrapper
-    mandoc
-    mdbook
-    meson
-    mime-types
-    minify
-    mkShell
-    moreutils
-    nano
-    ncurses
-    nginx
-    ninja
-    nix
-    ocamlPackages
-    openssh
-    openssl
-    overrideCC
-    pandoc
-    pango
-    parallel
-    path
-    pkgconfig
-    pkgsCross
-    plantuml
-    postgresql
-    pounce
-    pulseaudio
-    python3
-    python3Packages
-    quassel
-    remarshal
-    rink
-    ripgrep
-    rsync
-    runCommand
-    runCommandLocal
-    runCommandNoCC
-    rustPlatform
-    rustc
-    s6-portable-utils
-    sbcl
-    shellcheck
-    sqlite
-    stdenvNoCC
-    stern
-    substituteAll
-    symlinkJoin
-    systemd
-    tdlib
-    teensy-loader-cli
-    terraform_0_12
-    texlive
-    thttpd
-    tree
-    tree-sitter
-    unzip
-    which
-    writers
-    writeShellScript
-    writeShellScriptBin
-    writeText
-    xorg
-    xz
-    zlib
-    zstd;
-
-  # Inherit packages from the stable channel for things that are
-  # broken on unstable
-  inherit (stableNixpkgs)
-    awscli # TODO(grfn): Move back to unstable once it is fixed
-    ;
-
-  # Required by //third_party/nix
-  inherit (nixpkgs)
-    aws-sdk-cpp
-    bison
-    boehmgc
-    boost # urgh
-    brotli
-    busybox-sandbox-shell
-    curl
-    docbook5
-    docbook_xsl_ns
-    editline
-    flex
-    libseccomp
-    libsodium
-    libxml2
-    libxslt
-    mercurial
-    perl
-    perlPackages
-    ;
-
-  haskellPackages = (nixpkgs.haskellPackages.override {
-    overrides = (import ../../overlays/haskell
-    { /* empty readTree arg */ }
-    { pkgs = nixpkgs; });
-  });
-
-  gradle_6 = (nixpkgs.gradleGen.override {
-    java = nixpkgs.jdk11;
-    jdk = nixpkgs.jdk11;
-  }).gradleGen rec {
-    name = "gradle-6.5.1";
-    nativeVersion = "0.22-milestone-3";
-
-    src = builtins.fetchurl {
-      url = "https://services.gradle.org/distributions/${name}-bin.zip";
-      sha256 = "0jmmipjh4fbsn92zpifa5cqg5ws2a4ha0s4jzqhrg4zs542x79sh";
-    };
-  };
-}
diff --git a/third_party/nixpkgs/default.nix b/third_party/nixpkgs/default.nix
new file mode 100644
index 000000000000..53adfe81e388
--- /dev/null
+++ b/third_party/nixpkgs/default.nix
@@ -0,0 +1,56 @@
+# This file imports the pinned nixpkgs sets and applies relevant
+# modifications, such as our overlays.
+#
+# Note that the attribute exposed by this (third_party.nixpkgs) is
+# "special" in that the fixpoint used as readTree's config parameter
+# in //default.nix passes this attribute as the `pkgs` argument to all
+# readTree derivations.
+
+{ depot, ... }:
+
+let
+  # This provides the sources of nixpkgs. We track both
+  # nixos-unstable, and the current stable channel of the latest NixOS
+  # release.
+
+  # Tracking nixos-unstable as of 2021-03-25.
+  unstableHashes = {
+    commit = "60dd94fb7e01a8288f6638eee71d7cb354c49327";
+    sha256 = "0skdwk9bdld295kzrymirs8xrzycqmhsclaz8s18jhcz75hb8sk3";
+  };
+
+  # Tracking nixos-20.09 as of 2021-03-25.
+  stableHashes = {
+    commit = "223d0d733a66b46504ea6b4c15f88b7cc4db58fb";
+    sha256 = "073327ris0frqa3kpid3nsjr9w8yx2z83xpsc24w898mrs9r7d5v";
+  };
+
+  nixpkgsSrc = fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/${unstableHashes.commit}.tar.gz";
+    sha256 = unstableHashes.sha256;
+  };
+  stableNixpkgsSrc = fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/${stableHashes.commit}.tar.gz";
+    sha256 = stableHashes.sha256;
+  };
+
+  # Stable package set is imported, but not exposed, to overlay
+  # required packages into the unstable set.
+  stableNixpkgs = import stableNixpkgsSrc {};
+
+  # Overlay for packages that should come from the stable channel
+  # instead (e.g. because something is broken in unstable).
+  stableOverlay = self: super: {
+    inherit (stableNixpkgs)
+      awscli # TODO(grfn): Move back to unstable once it is fixed
+      ;
+  };
+in import nixpkgsSrc {
+  config.allowUnfree = true;
+  config.allowBroken = true;
+  overlays = [
+    stableOverlay
+    depot.third_party.overlays.tvl
+    depot.third_party.overlays.haskell
+  ];
+}