8 files changed, 627 insertions, 0 deletions
diff --git a/nix/buildGo/.skip-subtree b/nix/buildGo/.skip-subtree
new file mode 100644
index 000000000000..8db1f814f653
--- /dev/null
+++ b/nix/buildGo/.skip-subtree
@@ -0,0 +1,2 @@
+Subdirectories of this folder should not be imported since they are
+internal to buildGo.nix and incompatible with readTree.
diff --git a/nix/buildGo/README.md b/nix/buildGo/README.md
new file mode 100644
index 000000000000..e9667c039ab7
--- /dev/null
+++ b/nix/buildGo/README.md
@@ -0,0 +1,117 @@
+buildGo.nix
+===========
+
+This is an alternative [Nix][] build system for [Go][]. It supports building Go
+libraries and programs.
+
+*Note:* This will probably end up being folded into [Nixery][].
+
+## Background
+
+Most language-specific Nix tooling outsources the build to existing
+language-specific build tooling, which essentially means that Nix ends up being
+a wrapper around all sorts of external build systems.
+
+However, systems like [Bazel][] take an alternative approach in which the
+compiler is invoked directly and the composition of programs and libraries stays
+within a single homogeneous build system.
+
+Users don't need to learn per-language build systems and especially for
+companies with large monorepo-setups ([like Google][]) this has huge
+productivity impact.
+
+This project is an attempt to prove that Nix can be used in a similar style to
+build software directly, rather than shelling out to other build systems.
+
+## Example
+
+Given a program layout like this:
+
+```
+.
+├── lib          <-- some library component
+│   ├── bar.go
+│   └── foo.go
+├── main.go      <-- program implementation
+└── default.nix  <-- build instructions
+```
+
+The contents of `default.nix` could look like this:
+
+```nix
+{ buildGo }:
+
+let
+  lib = buildGo.package {
+    name = "somelib";
+    srcs = [
+      ./lib/bar.go
+      ./lib/foo.go
+    ];
+  };
+in buildGo.program {
+  name = "my-program";
+  deps = [ lib ];
+
+  srcs = [
+    ./main.go
+  ];
+}
+```
+
+(If you don't know how to read Nix, check out [nix-1p][])
+
+## Usage
+
+`buildGo` exposes five different functions:
+
+* `buildGo.program`: Build a Go binary out of the specified source files.
+
+  | parameter | type                    | use                                            | required? |
+  |-----------|-------------------------|------------------------------------------------|-----------|
+  | `name`    | `string`                | Name of the program (and resulting executable) | yes       |
+  | `srcs`    | `list<path>`            | List of paths to source files                  | yes       |
+  | `deps`    | `list<drv>`             | List of dependencies (i.e. other Go libraries) | no        |
+  | `x_defs`  | `attrs<string, string>` | Attribute set of linker vars (i.e. `-X`-flags) | no        |
+
+* `buildGo.package`: Build a Go library out of the specified source files.
+
+  | parameter | type         | use                                            | required? |
+  |-----------|--------------|------------------------------------------------|-----------|
+  | `name`    | `string`     | Name of the library                            | yes       |
+  | `srcs`    | `list<path>` | List of paths to source files                  | yes       |
+  | `deps`    | `list<drv>`  | List of dependencies (i.e. other Go libraries) | no        |
+  | `path`    | `string`     | Go import path for the resulting library       | no        |
+
+* `buildGo.external`: Build an externally defined Go library or program.
+
+  This function performs analysis on the supplied source code (which
+  can use the standard Go tooling layout) and creates a tree of all
+  the packages contained within.
+
+  This exists for compatibility with external libraries that were not
+  defined using buildGo.
+
+  | parameter | type           | use                                           | required? |
+  |-----------|----------------|-----------------------------------------------|-----------|
+  | `path`    | `string`       | Go import path for the resulting package      | yes       |
+  | `src`     | `path`         | Path to the source **directory**              | yes       |
+  | `deps`    | `list<drv>`    | List of dependencies (i.e. other Go packages) | no        |
+
+## Current status
+
+This project is work-in-progress. Crucially it is lacking the following features:
+
+* feature flag parity with Bazel's Go rules
+* documentation building
+* test execution
+
+There are still some open questions around how to structure some of those
+features in Nix.
+
+[Nix]: https://nixos.org/nix/
+[Go]: https://golang.org/
+[Nixery]: https://github.com/google/nixery
+[Bazel]: https://bazel.build/
+[like Google]: https://ai.google/research/pubs/pub45424
+[nix-1p]: https://github.com/tazjin/nix-1p
diff --git a/nix/buildGo/default.nix b/nix/buildGo/default.nix
new file mode 100644
index 000000000000..bd2e32330060
--- /dev/null
+++ b/nix/buildGo/default.nix
@@ -0,0 +1,121 @@
+# Copyright 2019 Google LLC.
+# SPDX-License-Identifier: Apache-2.0
+#
+# buildGo provides Nix functions to build Go packages in the style of Bazel's
+# rules_go.
+
+{ pkgs ? import <nixpkgs> { }
+, ...
+}:
+
+let
+  inherit (builtins)
+    attrNames
+    baseNameOf
+    dirOf
+    elemAt
+    filter
+    listToAttrs
+    map
+    match
+    readDir
+    replaceStrings
+    toString;
+
+  inherit (pkgs) lib runCommand fetchFromGitHub protobuf symlinkJoin;
+
+  # TODO: Adapt to Go 1.19 changes
+  go = pkgs.go_1_18;
+
+  # Helpers for low-level Go compiler invocations
+  spaceOut = lib.concatStringsSep " ";
+
+  includeDepSrc = dep: "-I ${dep}";
+  includeSources = deps: spaceOut (map includeDepSrc deps);
+
+  includeDepLib = dep: "-L ${dep}";
+  includeLibs = deps: spaceOut (map includeDepLib deps);
+
+  srcBasename = src: elemAt (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src)) 1;
+  srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}";
+  srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs);
+
+  allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps)));
+
+  xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs));
+
+  pathToName = p: replaceStrings [ "/" ] [ "_" ] (toString p);
+
+  # Add an `overrideGo` attribute to a function result that works
+  # similar to `overrideAttrs`, but is used specifically for the
+  # arguments passed to Go builders.
+  makeOverridable = f: orig: (f orig) // {
+    overrideGo = new: makeOverridable f (orig // (new orig));
+  };
+
+  # High-level build functions
+
+  # Build a Go program out of the specified files and dependencies.
+  program = { name, srcs, deps ? [ ], x_defs ? { } }:
+    let uniqueDeps = allDeps (map (d: d.gopkg) deps);
+    in runCommand name { } ''
+      ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs}
+      mkdir -p $out/bin
+      export GOROOT_FINAL=go
+      ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a
+    '';
+
+  # Build a Go library assembled out of the specified files.
+  #
+  # This outputs both the sources and compiled binary, as both are
+  # needed when downstream packages depend on it.
+  package = { name, srcs, deps ? [ ], path ? name, sfiles ? [ ] }:
+    let
+      uniqueDeps = allDeps (map (d: d.gopkg) deps);
+
+      # The build steps below need to be executed conditionally for Go
+      # assembly if the analyser detected any *.s files.
+      #
+      # This is required for several popular packages (e.g. x/sys).
+      ifAsm = do: lib.optionalString (sfiles != [ ]) do;
+      asmBuild = ifAsm ''
+        ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles}
+        ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles}
+      '';
+      asmLink = ifAsm "-symabis ./symabis -asmhdr $out/go_asm.h";
+      asmPack = ifAsm ''
+        ${go}/bin/go tool pack r $out/${path}.a ./asm.o
+      '';
+
+      gopkg = (runCommand "golib-${name}" { } ''
+        mkdir -p $out/${path}
+        ${srcList path (map (s: "${s}") srcs)}
+        ${asmBuild}
+        ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs}
+        ${asmPack}
+      '').overrideAttrs (_: {
+        passthru = {
+          inherit gopkg;
+          goDeps = uniqueDeps;
+          goImportPath = path;
+        };
+      });
+    in
+    gopkg;
+
+  # Build a tree of Go libraries out of an external Go source
+  # directory that follows the standard Go layout and was not built
+  # with buildGo.nix.
+  #
+  # The derivation for each actual package will reside in an attribute
+  # named "gopkg", and an attribute named "gobin" for binaries.
+  external = import ./external { inherit pkgs program package; };
+
+in
+{
+  # Only the high-level builder functions are exposed, but made
+  # overrideable.
+  program = makeOverridable program;
+  package = makeOverridable package;
+  external = makeOverridable external;
+}
diff --git a/nix/buildGo/example/default.nix b/nix/buildGo/example/default.nix
new file mode 100644
index 000000000000..6756bf39e20b
--- /dev/null
+++ b/nix/buildGo/example/default.nix
@@ -0,0 +1,40 @@
+# Copyright 2019 Google LLC.
+# SPDX-License-Identifier: Apache-2.0
+
+# This file provides examples for how to use the various builder
+# functions provided by `buildGo`.
+#
+# The features used in the example are not exhaustive, but should give
+# users a quick introduction to how to use buildGo.
+
+let
+  buildGo = import ../default.nix { };
+
+  # Example use of buildGo.package, which creates an importable Go
+  # package from the specified source files.
+  examplePackage = buildGo.package {
+    name = "example";
+    srcs = [
+      ./lib.go
+    ];
+  };
+
+  # Example use of buildGo.program, which builds an executable using
+  # the specified name and dependencies (which in turn must have been
+  # created via buildGo.package etc.)
+in
+buildGo.program {
+  name = "example";
+
+  srcs = [
+    ./main.go
+  ];
+
+  deps = [
+    examplePackage
+  ];
+
+  x_defs = {
+    "main.Flag" = "successfully";
+  };
+}
diff --git a/nix/buildGo/example/lib.go b/nix/buildGo/example/lib.go
new file mode 100644
index 000000000000..8a61370e994c
--- /dev/null
+++ b/nix/buildGo/example/lib.go
@@ -0,0 +1,9 @@
+// Copyright 2019 Google LLC.
+// SPDX-License-Identifier: Apache-2.0
+
+package example
+
+// UUID returns a totally random, carefully chosen UUID
+func UUID() string {
+	return "3640932f-ad40-4bc9-b45d-f504a0f5910a"
+}
diff --git a/nix/buildGo/example/main.go b/nix/buildGo/example/main.go
new file mode 100644
index 000000000000..bbcedbff8726
--- /dev/null
+++ b/nix/buildGo/example/main.go
@@ -0,0 +1,25 @@
+// Copyright 2019 Google LLC.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Package main provides a tiny example program for the Bazel-style
+// Nix build system for Go.
+
+package main
+
+import (
+	"example"
+	"exampleproto"
+	"fmt"
+)
+
+var Flag string = "unsuccessfully"
+
+func main() {
+	thing := exampleproto.Thing{
+		Id:          example.UUID(),
+		KindOfThing: "test thing",
+	}
+
+	fmt.Printf("The thing is a %s with ID %q\n", thing.Id, thing.KindOfThing)
+	fmt.Printf("The flag has been %s set\n", Flag)
+}
diff --git a/nix/buildGo/external/default.nix b/nix/buildGo/external/default.nix
new file mode 100644
index 000000000000..42592c67e482
--- /dev/null
+++ b/nix/buildGo/external/default.nix
@@ -0,0 +1,112 @@
+# Copyright 2019 Google LLC.
+# SPDX-License-Identifier: Apache-2.0
+{ pkgs, program, package }:
+
+let
+  inherit (builtins)
+    elemAt
+    foldl'
+    fromJSON
+    head
+    length
+    listToAttrs
+    readFile
+    replaceStrings
+    tail
+    unsafeDiscardStringContext
+    throw;
+
+  inherit (pkgs) lib runCommand go jq ripgrep;
+
+  pathToName = p: replaceStrings [ "/" ] [ "_" ] (toString p);
+
+  # Collect all non-vendored dependencies from the Go standard library
+  # into a file that can be used to filter them out when processing
+  # dependencies.
+  stdlibPackages = runCommand "stdlib-pkgs.json" { } ''
+    export HOME=$PWD
+    export GOPATH=/dev/null
+    ${go}/bin/go list std | \
+      ${ripgrep}/bin/rg -v 'vendor' | \
+      ${jq}/bin/jq -R '.' | \
+      ${jq}/bin/jq -c -s 'map({key: ., value: true}) | from_entries' \
+      > $out
+  '';
+
+  analyser = program {
+    name = "analyser";
+
+    srcs = [
+      ./main.go
+    ];
+
+    x_defs = {
+      "main.stdlibList" = "${stdlibPackages}";
+    };
+  };
+
+  mkset = path: value:
+    if path == [ ] then { gopkg = value; }
+    else { "${head path}" = mkset (tail path) value; };
+
+  last = l: elemAt l ((length l) - 1);
+
+  toPackage = self: src: path: depMap: entry:
+    let
+      localDeps = map
+        (d: lib.attrByPath (d ++ [ "gopkg" ])
+          (
+            throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'"
+          )
+          self)
+        entry.localDeps;
+
+      foreignDeps = map
+        (d: lib.attrByPath [ d.path ]
+          (
+            throw "missing foreign dependency '${d.path}' in '${path}, imported at ${d.position}'"
+          )
+          depMap)
+        entry.foreignDeps;
+
+      args = {
+        srcs = map (f: src + ("/" + f)) entry.files;
+        deps = localDeps ++ foreignDeps;
+      };
+
+      libArgs = args // {
+        name = pathToName entry.name;
+        path = lib.concatStringsSep "/" ([ path ] ++ entry.locator);
+        sfiles = map (f: src + ("/" + f)) entry.sfiles;
+      };
+
+      binArgs = args // {
+        name = (last ((lib.splitString "/" path) ++ entry.locator));
+      };
+    in
+    if entry.isCommand then (program binArgs) else (package libArgs);
+
+in
+{ src, path, deps ? [ ] }:
+let
+  # Build a map of dependencies (from their import paths to their
+  # derivation) so that they can be conditionally imported only in
+  # sub-packages that require them.
+  depMap = listToAttrs (map
+    (d: {
+      name = d.goImportPath;
+      value = d;
+    })
+    (map (d: d.gopkg) deps));
+
+  name = pathToName path;
+  analysisOutput = runCommand "${name}-structure.json" { } ''
+    ${analyser}/bin/analyser -path ${path} -source ${src} > $out
+  '';
+  # readFile adds the references of the read in file to the string context for
+  # Nix >= 2.6 which would break the attribute set construction in fromJSON
+  analysis = fromJSON (unsafeDiscardStringContext (readFile analysisOutput));
+in
+lib.fix (self: foldl' lib.recursiveUpdate { } (
+  map (entry: mkset entry.locator (toPackage self src path depMap entry)) analysis
+))
diff --git a/nix/buildGo/external/main.go b/nix/buildGo/external/main.go
new file mode 100644
index 000000000000..a77c43b371e9
--- /dev/null
+++ b/nix/buildGo/external/main.go
@@ -0,0 +1,201 @@
+// Copyright 2019 Google LLC.
+// SPDX-License-Identifier: Apache-2.0
+
+// This tool analyses external (i.e. not built with `buildGo.nix`) Go
+// packages to determine a build plan that Nix can import.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"go/build"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+// Path to a JSON file describing all standard library import paths.
+// This file is generated and set here by Nix during the build
+// process.
+var stdlibList string
+
+// pkg describes a single Go package within the specified source
+// directory.
+//
+// Return information includes the local (relative from project root)
+// and external (none-stdlib) dependencies of this package.
+type pkg struct {
+	Name        string       `json:"name"`
+	Locator     []string     `json:"locator"`
+	Files       []string     `json:"files"`
+	SFiles      []string     `json:"sfiles"`
+	LocalDeps   [][]string   `json:"localDeps"`
+	ForeignDeps []foreignDep `json:"foreignDeps"`
+	IsCommand   bool         `json:"isCommand"`
+}
+
+type foreignDep struct {
+	Path string `json:"path"`
+	// filename, column and line number of the import, if known
+	Position string `json:"position"`
+}
+
+// findGoDirs returns a filepath.WalkFunc that identifies all
+// directories that contain Go source code in a certain tree.
+func findGoDirs(at string) ([]string, error) {
+	dirSet := make(map[string]bool)
+
+	err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		name := info.Name()
+		// Skip folders that are guaranteed to not be relevant
+		if info.IsDir() && (name == "testdata" || name == ".git") {
+			return filepath.SkipDir
+		}
+
+		// If the current file is a Go file, then the directory is popped
+		// (i.e. marked as a Go directory).
+		if !info.IsDir() && strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go") {
+			dirSet[filepath.Dir(path)] = true
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	goDirs := []string{}
+	for k, _ := range dirSet {
+		goDirs = append(goDirs, k)
+	}
+
+	return goDirs, nil
+}
+
+// analysePackage loads and analyses the imports of a single Go
+// package, returning the data that is required by the Nix code to
+// generate a derivation for this package.
+func analysePackage(root, source, importpath string, stdlib map[string]bool) (pkg, error) {
+	ctx := build.Default
+	ctx.CgoEnabled = false
+
+	p, err := ctx.ImportDir(source, build.IgnoreVendor)
+	if err != nil {
+		return pkg{}, err
+	}
+
+	local := [][]string{}
+	foreign := []foreignDep{}
+
+	for _, i := range p.Imports {
+		if stdlib[i] {
+			continue
+		}
+
+		if i == importpath {
+			local = append(local, []string{})
+		} else if strings.HasPrefix(i, importpath+"/") {
+			local = append(local, strings.Split(strings.TrimPrefix(i, importpath+"/"), "/"))
+		} else {
+			// The import positions is a map keyed on the import name.
+			// The value is a list, presumably because an import can appear
+			// multiple times in a package. Let’s just take the first one,
+			// should be enough for a good error message.
+			firstPos := p.ImportPos[i][0].String()
+			foreign = append(foreign, foreignDep{Path: i, Position: firstPos})
+		}
+	}
+
+	prefix := strings.TrimPrefix(source, root+"/")
+
+	locator := []string{}
+	if len(prefix) != len(source) {
+		locator = strings.Split(prefix, "/")
+	} else {
+		// Otherwise, the locator is empty since its the root package and
+		// no prefix should be added to files.
+		prefix = ""
+	}
+
+	files := []string{}
+	for _, f := range p.GoFiles {
+		files = append(files, path.Join(prefix, f))
+	}
+
+	sfiles := []string{}
+	for _, f := range p.SFiles {
+		sfiles = append(sfiles, path.Join(prefix, f))
+	}
+
+	return pkg{
+		Name:        path.Join(importpath, prefix),
+		Locator:     locator,
+		Files:       files,
+		SFiles:      sfiles,
+		LocalDeps:   local,
+		ForeignDeps: foreign,
+		IsCommand:   p.IsCommand(),
+	}, nil
+}
+
+func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) {
+	f, err := ioutil.ReadFile(from)
+	if err != nil {
+		return
+	}
+
+	err = json.Unmarshal(f, &pkgs)
+	return
+}
+
+func main() {
+	source := flag.String("source", "", "path to directory with sources to process")
+	path := flag.String("path", "", "import path for the package")
+
+	flag.Parse()
+
+	if *source == "" {
+		log.Fatalf("-source flag must be specified")
+	}
+
+	stdlibPkgs, err := loadStdlibPkgs(stdlibList)
+	if err != nil {
+		log.Fatalf("failed to load standard library index from %q: %s\n", stdlibList, err)
+	}
+
+	goDirs, err := findGoDirs(*source)
+	if err != nil {
+		log.Fatalf("failed to walk source directory '%s': %s", *source, err)
+	}
+
+	all := []pkg{}
+	for _, d := range goDirs {
+		analysed, err := analysePackage(*source, d, *path, stdlibPkgs)
+
+		// If the Go source analysis returned "no buildable Go files",
+		// that directory should be skipped.
+		//
+		// This might be due to `+build` flags on the platform and other
+		// reasons (such as test files).
+		if _, ok := err.(*build.NoGoError); ok {
+			continue
+		}
+
+		if err != nil {
+			log.Fatalf("failed to analyse package at %q: %s", d, err)
+		}
+		all = append(all, analysed)
+	}
+
+	j, _ := json.Marshal(all)
+	fmt.Println(string(j))
+}