Merge pull request #9 from tazjin/feat/read-tree r/95

Configure automatic package layouts via repository structure

20 files changed, 191 insertions, 111 deletions

diff --git a/default.nix b/default.nix
index 7df654798fd7..d7ef5b72bcfc 100644
--- a/default.nix
+++ b/default.nix
@@ -14,88 +14,44 @@ let
     url = "https://github.com/NixOS/nixpkgs-channels/archive/${stableCommit}.tar.gz";
     sha256 = "0243qiivxl3z51biy4f5y5cy81x5bki5dazl9wqwgnmd373gpmxy";
   };
-
-  unstableCommit = "765a71f15025ce78024bae3dc4a92bd2be3a8fbf";
-  unstableSrc = fetchTarball {
-    url = "https://github.com/NixOS/nixpkgs-channels/archive/${unstableCommit}.tar.gz";
-    sha256 = "0j1wghr9dj9njn3x9xi0wzjk1107gi2pxb0w2dk8g0djmhnlx71q";
-  };
-  unstable = import unstableSrc {};
-
-  localPkgs = self: super: {
-    # Local projects should be added here:
-    tazjin = {
-      blog = self.callPackage ./services/tazblog {};
-      blog_cli = self.callPackage ./tools/blog_cli {};
-      gemma = self.callPackage ./services/gemma {};
-      nixcon = self.naersk.buildPackage ./services/nixcon-demo {};
-
-      kms_pass = self.callPackage ./tools/kms_pass {
+  readTree = import ./read-tree.nix;
+
+  # Derivations that have `meta.enableCI` set to `true` should be
+  # built by the CI system on every commit. This code implements
+  # filtering of all derivations in the local sets against this
+  # condition.
+  filterCI = lib: pkgs: let
+    inherit (lib) collect isDerivation filterAttrsRecursive;
+    ciCondition = _: x: (!isDerivation x) || ((x ? meta.enableCI) && (x.meta.enableCI));
+  in collect isDerivation (filterAttrsRecursive ciCondition pkgs);
+
+  repoPkgs = self: super:
+    let config = {
+      pkgs = self;
+      upstream = super;
+
+      kms = {
         project = "tazjins-infrastructure";
         region = "europe-north1";
         keyring = "tazjins-keys";
         key = "kontemplate-key";
       };
     };
-
-    # Third-party projects (either vendored or modified from nixpkgs) go here:
-    nixery = import ./third_party/nixery.nix { pkgs = self; };
-    terraform-gcp = self.terraform_0_12.withPlugins(p: [ p.google p.google-beta ]);
-    ormolu = import (self.fetchFromGitHub {
-      owner = "tweag";
-      repo = "ormolu";
-      rev = "a7076c0f83e5c06ea9067b71171859fa2ba8afd9";
-      sha256 = "1p4n2ja4ciw3qfskn65ggpy37mvgf2sslxqmqn8s8jjarnqcyfny";
-    }) { pkgs = self; };
-    naersk = self.callPackage (self.fetchFromGitHub {
-      owner = "nmattia";
-      repo = "naersk";
-      rev = "68c1c2b2b661913cdc5ecabea518dfdc4f449027";
-      sha256 = "1ll310pl44kdbwfslzwvg2v7khf1y0xkg2j5wcfia4k7sj6bcl28";
-    }) {};
-
-    # Gemma needs an older version of Elm to be built. Updating it to
-    # the newer version is a lot of effort.
-    elmPackages = (import (self.fetchFromGitHub {
-      owner = "NixOS";
-      repo = "nixpkgs";
-      rev = "14f9ee66e63077539252f8b4550049381a082518";
-      sha256 = "1wn7nmb1cqfk2j91l3rwc6yhimfkzxprb8wknw5wi57yhq9m6lv1";
-    }) {}).elmPackages;
-
-    # Wrap kontemplate to inject the Cloud KMS version of 'pass'
-    kontemplate =
-      let master = super.kontemplate.overrideAttrs(_: {
-        src = self.fetchFromGitHub {
-          owner = "tazjin";
-          repo = "kontemplate";
-          rev = "v1.8.0";
-          sha256 = "123mjmmm4hynraq1fpn3j5i0a1i87l265kkjraxxxbl0zacv74i1";
-        };
-      });
-      in self.writeShellScriptBin "kontemplate" ''
-        export PATH="${self.tazjin.kms_pass}/bin:$PATH"
-        exec ${master}/bin/kontemplate $@
-      '';
-
-    # One of Gemma's dependencies is missing in nixpkgs' Quicklisp
-    # package set, it is overlaid locally here.
-    lispPackages = import ./third_party/common_lisp/quicklisp.nix {
-      inherit (self) lib;
-      inherit (super) lispPackages;
+    in {
+      services = readTree ./services config;
+      tools = readTree ./tools config;
+      third_party = readTree ./third_party config;
+    }
+    # Load overrides into the top-level:
+    // (readTree ./overrides config)
+    # Collect all projects that should be built by CI
+    // {
+      ciProjects = (filterCI super.lib self.services)
+        ++ (filterCI super.lib self.tools)
+        ++ (filterCI super.lib self.third_party);
     };
-
-    # All projects that should be built by CI should be added here:
-    ciProjects = [
-      self.kontemplate
-      self.nixery
-      self.ormolu
-      self.terraform-gcp
-    ] ++ filter (d: d ? meta.broken && !d.meta.broken) (attrValues self.tazjin);
-  };
-
 in { ... } @ args: import stableSrc (args // {
-    overlays = [ localPkgs ];
+    overlays = [ repoPkgs ];
     config.allowUnfree = true;
     config.allowBroken = true;
 })
diff --git a/infra/kubernetes/tazblog/config.yaml b/infra/kubernetes/tazblog/config.yaml
index 1ab6e9d2b421..165a30f6839b 100644
--- a/infra/kubernetes/tazblog/config.yaml
+++ b/infra/kubernetes/tazblog/config.yaml
@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
       - name: tazblog
-        image: nixery.local/shell/tazjin.blog:{{ gitHEAD }}
+        image: nixery.local/shell/services.tazblog:{{ gitHEAD }}
         command: [ "tazblog" ]
 ---
 apiVersion: v1
diff --git a/overrides/elmPackages.nix b/overrides/elmPackages.nix
new file mode 100644
index 000000000000..3df44420a6bb
--- /dev/null
+++ b/overrides/elmPackages.nix
@@ -0,0 +1,10 @@
+# Gemma needs an older version of Elm to be built. Updating it to
+# the newer version is a lot of effort.
+{ pkgs, ... }:
+
+(import (pkgs.fetchFromGitHub {
+  owner = "NixOS";
+  repo = "nixpkgs";
+  rev = "14f9ee66e63077539252f8b4550049381a082518";
+  sha256 = "1wn7nmb1cqfk2j91l3rwc6yhimfkzxprb8wknw5wi57yhq9m6lv1";
+}) {}).elmPackages
diff --git a/overrides/kontemplate.nix b/overrides/kontemplate.nix
new file mode 100644
index 000000000000..28381b0137f8
--- /dev/null
+++ b/overrides/kontemplate.nix
@@ -0,0 +1,14 @@
+{ pkgs, upstream, ... }:
+
+let master = upstream.kontemplate.overrideAttrs(_: {
+  src = pkgs.fetchFromGitHub {
+    owner = "tazjin";
+    repo = "kontemplate";
+    rev = "v1.8.0";
+    sha256 = "123mjmmm4hynraq1fpn3j5i0a1i87l265kkjraxxxbl0zacv74i1";
+  };
+});
+in pkgs.writeShellScriptBin "kontemplate" ''
+  export PATH="${pkgs.tools.kms_pass}/bin:$PATH"
+  exec ${master}/bin/kontemplate $@
+''
diff --git a/overrides/lispPackages/default.nix b/overrides/lispPackages/default.nix
new file mode 100644
index 000000000000..da8f3c893ae7
--- /dev/null
+++ b/overrides/lispPackages/default.nix
@@ -0,0 +1,8 @@
+# One of Gemma's dependencies is missing in nixpkgs' Quicklisp
+# package set, it is overlaid locally here.
+{ pkgs, upstream, ... }:
+
+import ./quicklisp.nix {
+  inherit (pkgs) lib;
+  inherit (upstream) lispPackages;
+}
diff --git a/third_party/common_lisp/quicklisp-to-nix-output/cl-prevalence.nix b/overrides/lispPackages/quicklisp-to-nix-output/cl-prevalence.nix
index 4e5e3ec5d6e8..4e5e3ec5d6e8 100644
--- a/third_party/common_lisp/quicklisp-to-nix-output/cl-prevalence.nix
+++ b/overrides/lispPackages/quicklisp-to-nix-output/cl-prevalence.nix
diff --git a/third_party/common_lisp/quicklisp-to-nix-output/s-sysdeps.nix b/overrides/lispPackages/quicklisp-to-nix-output/s-sysdeps.nix
index 1c28ec6e2afd..1c28ec6e2afd 100644
--- a/third_party/common_lisp/quicklisp-to-nix-output/s-sysdeps.nix
+++ b/overrides/lispPackages/quicklisp-to-nix-output/s-sysdeps.nix
diff --git a/third_party/common_lisp/quicklisp-to-nix-output/s-xml.nix b/overrides/lispPackages/quicklisp-to-nix-output/s-xml.nix
index ec12dde52231..ec12dde52231 100644
--- a/third_party/common_lisp/quicklisp-to-nix-output/s-xml.nix
+++ b/overrides/lispPackages/quicklisp-to-nix-output/s-xml.nix
diff --git a/third_party/common_lisp/quicklisp.nix b/overrides/lispPackages/quicklisp.nix
index a0040f1d6f7e..1d23db762d34 100644
--- a/third_party/common_lisp/quicklisp.nix
+++ b/overrides/lispPackages/quicklisp.nix
@@ -1,6 +1,3 @@
-# Overlay over `pkgs.lispPackages` that adds additional packages which
-# are missing from the imported Quicklisp package set in nixpkgs.
-
 { lib, lispPackages }:
 
 let inherit (lispPackages) buildLispPackage qlOverrides fetchurl;
diff --git a/read-tree.nix b/read-tree.nix
new file mode 100644
index 000000000000..d883d12c8171
--- /dev/null
+++ b/read-tree.nix
@@ -0,0 +1,65 @@
+path: { pkgs, ... } @ args:
+
+let
+  inherit (builtins)
+    attrNames
+    attrValues
+    filter
+    head
+    isString
+    listToAttrs
+    map
+    match
+    readDir
+    tail
+    toPath
+    toString;
+
+  zipAttrs = names: values:
+    if (names == []) || (values == [])
+    then []
+    else [{
+      name = head names;
+      value = head values;
+    }] ++ zipAttrs (tail names) (tail values);
+
+  attrsToList = attrs: zipAttrs (attrNames attrs) (attrValues attrs);
+
+  isFile = s: s == "regular";
+  isDir = s: s == "directory";
+
+  joinPath = p: f: toPath ((toString p) + "/" + f);
+
+  isNixFile = file:
+    let res = match "(.*)\.nix" file;
+    in if res == null then null else head res;
+
+  filterNixFiles = dir:
+    let files = filter (e: isFile e.value) dir;
+        nixFiles = map (f: {
+          # Name and value are intentionally flipped to get the
+          # correct attribute set structure back out
+          name = isNixFile f.name;
+          value = f.name; # i.e. the path
+        }) files;
+    in filter (f: isString f.name) nixFiles;
+
+  traverse = path: dir:
+    let nixFiles = filterNixFiles dir;
+        imported = map (f: {
+          inherit (f) name;
+          value = import (joinPath path f.value) args;
+        }) nixFiles;
+        dirs = map (d: {
+          inherit (d) name;
+          value = readTree (joinPath path d.name);
+        }) (filter (e: isDir e.value) dir);
+    in listToAttrs (imported ++ dirs);
+
+  importOr = path: dir: f:
+    if dir ? "default.nix"
+      then import path args
+      else f path (attrsToList dir);
+
+  readTree = path: importOr path (readDir path) traverse;
+in readTree path
diff --git a/services/gemma/default.nix b/services/gemma/default.nix
index 409d8eef381a..ea10a4c7d02e 100644
--- a/services/gemma/default.nix
+++ b/services/gemma/default.nix
@@ -1,17 +1,20 @@
-{ stdenv, sbcl, lispPackages, elmPackages, makeWrapper, openssl }:
-
-let frontend = stdenv.mkDerivation {
-  name = "gemma-frontend";
-  src = ./frontend;
-  buildInputs = [ elmPackages.elm ];
-
-  phases = [ "unpackPhase" "buildPhase" ];
-  buildPhase = ''
-    mkdir .home && export HOME="$PWD/.home"
-    mkdir -p $out
-    elm-make --yes Main.elm --output $out/index.html
-  '';
-};
+{ pkgs, ... }:
+
+let
+  inherit (pkgs) stdenv sbcl lispPackages elmPackages makeWrapper openssl;
+
+  frontend = stdenv.mkDerivation {
+    name = "gemma-frontend";
+    src = ./frontend;
+    buildInputs = [ elmPackages.elm ];
+
+    phases = [ "unpackPhase" "buildPhase" ];
+    buildPhase = ''
+      mkdir .home && export HOME="$PWD/.home"
+      mkdir -p $out
+      elm-make --yes Main.elm --output $out/index.html
+    '';
+  };
 in stdenv.mkDerivation rec {
   name = "gemma";
   src = ./.;
diff --git a/services/nixcon-demo/default.nix b/services/nixcon-demo/default.nix
index e69de29bb2d1..0f4a330f7f74 100644
--- a/services/nixcon-demo/default.nix
+++ b/services/nixcon-demo/default.nix
@@ -0,0 +1,3 @@
+{ pkgs, ... }:
+
+pkgs.third_party.naersk.buildPackage ./. {}
diff --git a/services/tazblog/default.nix b/services/tazblog/default.nix
index 5dc3bdaf3eda..4d9608838d7c 100644
--- a/services/tazblog/default.nix
+++ b/services/tazblog/default.nix
@@ -2,9 +2,10 @@
 #
 # tazblog.nix was generated using cabal2nix.
 
-{ writeShellScriptBin, haskell }:
+{ pkgs, ... }:
 
 let
+  inherit (pkgs) writeShellScriptBin haskell;
   tazblog = haskell.packages.ghc865.callPackage ./tazblog.nix {};
   wrapper =  writeShellScriptBin "tazblog" ''
     export PORT=8000
@@ -13,4 +14,5 @@ let
   '';
 in wrapper.overrideAttrs(_: {
   allowSubstitutes = true;
+  meta.enableCI = true;
 })
diff --git a/third_party/naersk.nix b/third_party/naersk.nix
new file mode 100644
index 000000000000..c12c1abbbfa0
--- /dev/null
+++ b/third_party/naersk.nix
@@ -0,0 +1,9 @@
+{ pkgs, ... }:
+
+let inherit (pkgs) callPackage fetchFromGitHub;
+in callPackage (fetchFromGitHub {
+  owner = "nmattia";
+  repo = "naersk";
+  rev = "68c1c2b2b661913cdc5ecabea518dfdc4f449027";
+  sha256 = "1ll310pl44kdbwfslzwvg2v7khf1y0xkg2j5wcfia4k7sj6bcl28";
+}) {}
diff --git a/third_party/nixery.nix b/third_party/nixery.nix
index cb10e0b913b1..f778e5da13f9 100644
--- a/third_party/nixery.nix
+++ b/third_party/nixery.nix
@@ -1,6 +1,6 @@
 # Technically I suppose Nixery is not a third-party program, but it's
 # outside of this repository ...
-{ pkgs }:
+{ pkgs, ... }:
 
 let src = pkgs.fetchFromGitHub {
   owner = "google";
diff --git a/third_party/ormolu.nix b/third_party/ormolu.nix
new file mode 100644
index 000000000000..3175e25ff138
--- /dev/null
+++ b/third_party/ormolu.nix
@@ -0,0 +1,8 @@
+{ pkgs, ... }:
+
+import (pkgs.fetchFromGitHub {
+  owner = "tweag";
+  repo = "ormolu";
+  rev = "a7076c0f83e5c06ea9067b71171859fa2ba8afd9";
+  sha256 = "1p4n2ja4ciw3qfskn65ggpy37mvgf2sslxqmqn8s8jjarnqcyfny";
+}) { inherit pkgs; }
diff --git a/third_party/terraform-gcp.nix b/third_party/terraform-gcp.nix
new file mode 100644
index 000000000000..3332c12e41bc
--- /dev/null
+++ b/third_party/terraform-gcp.nix
@@ -0,0 +1,3 @@
+{ pkgs, ... }:
+
+pkgs.terraform_0_12.withPlugins(p: [ p.google p.google-beta ])
diff --git a/tools/bin/__dispatch.sh b/tools/bin/__dispatch.sh
index 20848bd5118c..c22b0339fd9e 100755
--- a/tools/bin/__dispatch.sh
+++ b/tools/bin/__dispatch.sh
@@ -11,19 +11,19 @@ readonly TARGET_TOOL=$(basename $0)
 
 case "${TARGET_TOOL}" in
   terraform)
-    attr="terraform-gcp"
+    attr="third_party.terraform-gcp"
     ;;
   kontemplate)
     attr="kontemplate"
     ;;
   blog_cli)
-    attr="tazjin.blog_cli"
+    attr="tools.blog_cli"
     ;;
   stern)
     attr="stern"
     ;;
   pass)
-    attr="tazjin.kms_pass"
+    attr="tools.kms_pass"
     ;;
   *)
     echo "The tool '${TARGET_TOOL}' is currently not installed in this repository."
diff --git a/tools/blog_cli/default.nix b/tools/blog_cli/default.nix
index c755d273a2b0..717daec86b9f 100644
--- a/tools/blog_cli/default.nix
+++ b/tools/blog_cli/default.nix
@@ -1,8 +1,10 @@
-{ buildGoPackage }:
+{ pkgs, ... }:
 
-buildGoPackage {
+pkgs.buildGoPackage {
   name = "blog_cli";
   goPackagePath = "github.com/tazjin/personal/blog_cli";
   src = ./.;
   goDeps = ./deps.nix;
+
+  meta.enableCI = true;
 }
diff --git a/tools/kms_pass/default.nix b/tools/kms_pass.nix
index fbc17650a948..7005697daaf8 100644
--- a/tools/kms_pass/default.nix
+++ b/tools/kms_pass.nix
@@ -6,10 +6,10 @@
 #
 # Only the 'show' and 'insert' commands are supported.
 
-{ google-cloud-sdk, tree, writeShellScriptBin
-, project, region, keyring, key }:
+{ pkgs, kms, ... }:
 
-writeShellScriptBin "pass" ''
+let inherit (pkgs) google-cloud-sdk tree writeShellScriptBin;
+in (writeShellScriptBin "pass" ''
   set -eo pipefail
 
   CMD="$1"
@@ -34,20 +34,20 @@ writeShellScriptBin "pass" ''
     show)
       secret_check
       ${google-cloud-sdk}/bin/gcloud kms decrypt \
-        --project ${project} \
-        --location ${region} \
-        --keyring ${keyring} \
-        --key ${key} \
+        --project ${kms.project} \
+        --location ${kms.region} \
+        --keyring ${kms.keyring} \
+        --key ${kms.key} \
         --ciphertext-file $SECRET_PATH \
         --plaintext-file -
       ;;
     insert)
       secret_check
       ${google-cloud-sdk}/bin/gcloud kms encrypt \
-        --project ${project} \
-        --location ${region} \
-        --keyring ${keyring} \
-        --key ${key} \
+        --project ${kms.project} \
+        --location ${kms.region} \
+        --keyring ${kms.keyring} \
+        --key ${kms.key} \
         --ciphertext-file $SECRET_PATH \
         --plaintext-file -
       echo "Inserted secret '$SECRET'"
@@ -57,4 +57,4 @@ writeShellScriptBin "pass" ''
       exit 1
       ;;
   esac
-''
+'') // { meta.enableCI = true; }