feat(external): Implement tool to analyse external dependencies

Adds a tool that can analyse dependencies that were not originally
meant to be built with buildGo.nix and return information that can be
used to construct appropriate Nix dependencies.

The tool will return information about package-local and foreign
dependencies separately to let Nix determine whether all required
dependencies are provided and to correctly link together sub-packages.

To avoid listing standard library imports in the dependencies, a list
of all packages in the standard library is generated statically to
allow for those to be filtered out during the analysis.

This tool is still work-in-progress.

2 files changed, 188 insertions, 0 deletions

diff --git a/external/default.nix b/external/default.nix
new file mode 100644
index 000000000000..79d559c05498
--- /dev/null
+++ b/external/default.nix
@@ -0,0 +1,29 @@
+# Copyright 2019 Google LLC.
+# SPDX-License-Identifier: Apache-2.0
+{ runCommand, go, jq, ripgrep, program }:
+
+let
+  # Collect all non-vendored dependencies from the Go standard library
+  # into a file that can be used to filter them out when processing
+  # dependencies.
+  stdlibPackages = runCommand "stdlib-pkgs.json" {} ''
+    export GOPATH=/dev/null
+    ${go}/bin/go list all | \
+      ${ripgrep}/bin/rg -v 'vendor' | \
+      ${jq}/bin/jq -R '.' | \
+      ${jq}/bin/jq -c -s 'map({key: ., value: true}) | from_entries' \
+      > $out
+  '';
+
+  analyser = program {
+    name = "analyser";
+
+    srcs = [
+      ./main.go
+    ];
+
+    x_defs = {
+      "main.stdlibList" = "${stdlibPackages}";
+    };
+  };
+in analyser
diff --git a/external/main.go b/external/main.go
new file mode 100644
index 000000000000..0f8834fc99c9
--- /dev/null
+++ b/external/main.go
@@ -0,0 +1,159 @@
+// Copyright 2019 Google LLC.
+// SPDX-License-Identifier: Apache-2.0
+
+// This tool analyses external (i.e. not built with `buildGo.nix`) Go
+// packages to determine a build plan that Nix can import.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"go/build"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// Path to a JSON file describing all standard library import paths.
+// This file is generated and set here by Nix during the build
+// process.
+var stdlibList string
+
+// pkg describes a single Go package within the specified source
+// directory.
+//
+// Return information includes the local (relative from project root)
+// and external (none-stdlib) dependencies of this package.
+type pkg struct {
+	Name        string   `json:"name"`
+	Source      string   `json:"source"`
+	Files       []string `json:"files"`
+	LocalDeps   []string `json:"localDeps"`
+	ForeignDeps []string `json:"foreignDeps"`
+}
+
+// findGoDirs returns a filepath.WalkFunc that identifies all
+// directories that contain Go source code in a certain tree.
+func findGoDirs(at string) ([]string, error) {
+	var goDirs []string
+	dir := ""
+
+	err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error {
+		// Skip testdata
+		if info.IsDir() && info.Name() == "testdata" {
+			return filepath.SkipDir
+		}
+
+		// Keep track of the last seen directory.
+		if info.IsDir() {
+			dir = path
+			return nil
+		}
+
+		// If the directory has already been "popped", nothing else needs
+		// to happen.
+		if dir == "" {
+			return nil
+		}
+
+		// If the current file is a Go file, then the directory is popped
+		// (i.e. marked as a Go directory).
+		if strings.HasSuffix(info.Name(), ".go") && !strings.HasSuffix(info.Name(), "_test.go") {
+			goDirs = append(goDirs, dir)
+			dir = ""
+			return nil
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return goDirs, nil
+}
+
+// analysePackage loads and analyses the imports of a single Go
+// package, returning the data that is required by the Nix code to
+// generate a derivation for this package.
+func analysePackage(root, source, path string, stdlib map[string]bool) (pkg, error) {
+	ctx := build.Default
+
+	p, err := ctx.ImportDir(source, build.IgnoreVendor)
+	if err != nil {
+		return pkg{}, err
+	}
+
+	local := []string{}
+	foreign := []string{}
+
+	for _, i := range p.Imports {
+		if stdlib[i] {
+			continue
+		}
+
+		if strings.HasPrefix(i, path) {
+			local = append(local, i)
+		} else {
+			foreign = append(foreign, i)
+		}
+	}
+
+	analysed := pkg{
+		Name:        strings.TrimPrefix(source, root+"/"),
+		Source:      source,
+		Files:       p.GoFiles,
+		LocalDeps:   local,
+		ForeignDeps: foreign,
+	}
+
+	return analysed, nil
+}
+
+func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) {
+	f, err := ioutil.ReadFile(from)
+	if err != nil {
+		return
+	}
+
+	err = json.Unmarshal(f, &pkgs)
+	return
+}
+
+func main() {
+	// TODO(tazjin): Remove default values
+	source := flag.String("source", "/nix/store/fzp67ris29zg5zfs65z0q245x0fahgll-source", "path to directory with sources to process")
+	path := flag.String("path", "github.com/golang/protobuf", "import path for the package")
+
+	flag.Parse()
+
+	if *source == "" {
+		log.Fatalf("-source flag must be specified")
+	}
+
+	stdlibPkgs, err := loadStdlibPkgs(stdlibList)
+	if err != nil {
+		log.Fatalf("failed to load standard library index from %q: %s\n", stdlibList, err)
+	}
+
+	goDirs, err := findGoDirs(*source)
+	if err != nil {
+		log.Fatalf("failed to walk source directory '%s': %s\n", source, err)
+	}
+
+	all := []pkg{}
+	for _, d := range goDirs {
+		analysed, err := analysePackage(*source, d, *path, stdlibPkgs)
+		if err != nil {
+			log.Fatalf("failed to analyse package at %q: %s", d, err)
+		}
+		all = append(all, analysed)
+	}
+
+	j, _ := json.MarshalIndent(all, "", "  ") // TODO: no indent
+	fmt.Println(string(j))
+}