diff options
Diffstat (limited to 'overrides')
| -rw-r--r-- | overrides/buildGo/.skip-subtree | 2 | ||||
| -rw-r--r-- | overrides/buildGo/README.md | 140 | ||||
| -rw-r--r-- | overrides/buildGo/default.nix | 128 | ||||
| -rw-r--r-- | overrides/buildGo/example/default.nix | 47 | ||||
| -rw-r--r-- | overrides/buildGo/example/lib.go | 9 | ||||
| -rw-r--r-- | overrides/buildGo/example/main.go | 25 | ||||
| -rw-r--r-- | overrides/buildGo/example/thing.proto | 10 | ||||
| -rw-r--r-- | overrides/buildGo/external/default.nix | 95 | ||||
| -rw-r--r-- | overrides/buildGo/external/main.go | 186 | ||||
| -rw-r--r-- | overrides/buildGo/proto.nix | 84 |
10 files changed, 0 insertions, 726 deletions
diff --git a/overrides/buildGo/.skip-subtree b/overrides/buildGo/.skip-subtree deleted file mode 100644 index 8db1f814f653..000000000000 --- a/overrides/buildGo/.skip-subtree +++ /dev/null @@ -1,2 +0,0 @@ -Subdirectories of this folder should not be imported since they are -internal to buildGo.nix and incompatible with readTree. diff --git a/overrides/buildGo/README.md b/overrides/buildGo/README.md deleted file mode 100644 index e84ede663bf8..000000000000 --- a/overrides/buildGo/README.md +++ /dev/null @@ -1,140 +0,0 @@ -buildGo.nix -=========== - -This is an alternative [Nix][] build system for [Go][]. It supports building Go -libraries and programs, and even automatically generating Protobuf & gRPC -libraries. - -*Note:* This will probably end up being folded into [Nixery][]. - -## Background - -Most language-specific Nix tooling outsources the build to existing -language-specific build tooling, which essentially means that Nix ends up being -a wrapper around all sorts of external build systems. - -However, systems like [Bazel][] take an alternative approach in which the -compiler is invoked directly and the composition of programs and libraries stays -within a single homogeneous build system. - -Users don't need to learn per-language build systems and especially for -companies with large monorepo-setups ([like Google][]) this has huge -productivity impact. - -This project is an attempt to prove that Nix can be used in a similar style to -build software directly, rather than shelling out to other build systems. - -## Example - -Given a program layout like this: - -``` -. -├── lib <-- some library component -│ ├── bar.go -│ └── foo.go -├── api.proto <-- gRPC API definition -├── main.go <-- program implementation -└── default.nix <-- build instructions -``` - -The contents of `default.nix` could look like this: - -```nix -{ buildGo }: - -let - api = buildGo.grpc { - name = "someapi"; - proto = ./api.proto; - }; - - lib = buildGo.package { - name = "somelib"; - srcs = [ - ./lib/bar.go - ./lib/foo.go - ]; - }; -in buildGo.program { - name = "my-program"; - deps = [ api lib ]; - - srcs = [ - ./main.go - ]; -} -``` - -(If you don't know how to read Nix, check out [nix-1p][]) - -## Usage - -`buildGo` exposes five different functions: - -* `buildGo.program`: Build a Go binary out of the specified source files. - - | parameter | type | use | required? | - |-----------|-------------------------|------------------------------------------------|-----------| - | `name` | `string` | Name of the program (and resulting executable) | yes | - | `srcs` | `list<path>` | List of paths to source files | yes | - | `deps` | `list<drv>` | List of dependencies (i.e. other Go libraries) | no | - | `x_defs` | `attrs<string, string>` | Attribute set of linker vars (i.e. `-X`-flags) | no | - -* `buildGo.package`: Build a Go library out of the specified source files. - - | parameter | type | use | required? | - |-----------|--------------|------------------------------------------------|-----------| - | `name` | `string` | Name of the library (and resulting executable) | yes | - | `srcs` | `list<path>` | List of paths to source files | yes | - | `deps` | `list<drv>` | List of dependencies (i.e. other Go libraries) | no | - | `path` | `string` | Go import path for the resulting library | no | - -* `buildGo.external`: Build an externally defined Go library or program. - - This function performs analysis on the supplied source code (which - can use the standard Go tooling layout) and creates a tree of all - the packages contained within. - - This exists for compatibility with external libraries that were not - defined using buildGo. - - | parameter | type | use | required? | - |-----------|----------------|-----------------------------------------------|-----------| - | `path` | `string` | Go import path for the resulting package | yes | - | `src` | `path` | Path to the source **directory** | yes | - | `deps` | `list<drv>` | List of dependencies (i.e. other Go packages) | no | - - For some examples of how `buildGo.external` is used, check out - [`proto.nix`](./proto.nix). - -* `buildGo.proto`: Build a Go library out of the specified Protobuf definition. - - | parameter | type | use | required? | - |-------------|-------------|--------------------------------------------------|-----------| - | `name` | `string` | Name for the resulting library | yes | - | `proto` | `path` | Path to the Protobuf definition file | yes | - | `path` | `string` | Import path for the resulting Go library | no | - | `extraDeps` | `list<drv>` | Additional Go dependencies to add to the library | no | - -* `buildGo.grpc`: Build a Go library out of the specified gRPC definition. - - The parameters are identical to `buildGo.proto`. - -## Current status - -This project is work-in-progress. Crucially it is lacking the following features: - -* feature flag parity with Bazel's Go rules -* documentation building -* test execution - -There are still some open questions around how to structure some of those -features in Nix. - -[Nix]: https://nixos.org/nix/ -[Go]: https://golang.org/ -[Nixery]: https://github.com/google/nixery -[Bazel]: https://bazel.build/ -[like Google]: https://ai.google/research/pubs/pub45424 -[nix-1p]: https://github.com/tazjin/nix-1p diff --git a/overrides/buildGo/default.nix b/overrides/buildGo/default.nix deleted file mode 100644 index 140cbf2d9d16..000000000000 --- a/overrides/buildGo/default.nix +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2019 Google LLC. -# SPDX-License-Identifier: Apache-2.0 -# -# buildGo provides Nix functions to build Go packages in the style of Bazel's -# rules_go. - -{ pkgs ? import <nixpkgs> {} -, ... }: - -let - inherit (builtins) - attrNames - baseNameOf - dirOf - elemAt - filter - listToAttrs - map - match - readDir - replaceStrings - toString; - - inherit (pkgs) lib go runCommand fetchFromGitHub protobuf symlinkJoin; - - # Helpers for low-level Go compiler invocations - spaceOut = lib.concatStringsSep " "; - - includeDepSrc = dep: "-I ${dep}"; - includeSources = deps: spaceOut (map includeDepSrc deps); - - includeDepLib = dep: "-L ${dep}"; - includeLibs = deps: spaceOut (map includeDepLib deps); - - srcBasename = src: elemAt (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src)) 1; - srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}"; - srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs); - - allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps))); - - xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); - - pathToName = p: replaceStrings ["/"] ["_"] (toString p); - - # Add an `overrideGo` attribute to a function result that works - # similar to `overrideAttrs`, but is used specifically for the - # arguments passed to Go builders. - makeOverridable = f: orig: (f orig) // { - overrideGo = new: makeOverridable f (orig // (new orig)); - }; - - # High-level build functions - - # Build a Go program out of the specified files and dependencies. - program = { name, srcs, deps ? [], x_defs ? {} }: - let uniqueDeps = allDeps deps; - in runCommand name {} '' - ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs} - mkdir -p $out/bin - ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a - ''; - - # Build a Go library assembled out of the specified files. - # - # This outputs both the sources and compiled binary, as both are - # needed when downstream packages depend on it. - package = { name, srcs, deps ? [], path ? name, sfiles ? [] }: - let - uniqueDeps = allDeps deps; - - # The build steps below need to be executed conditionally for Go - # assembly if the analyser detected any *.s files. - # - # This is required for several popular packages (e.g. x/sys). - ifAsm = do: if sfiles == [] then "" else do; - asmBuild = ifAsm '' - ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles} - ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles} - ''; - asmLink = ifAsm "-symabis ./symabis -asmhdr $out/go_asm.h"; - asmPack = ifAsm '' - ${go}/bin/go tool pack r $out/${path}.a ./asm.o - ''; - in (runCommand "golib-${name}" {} '' - mkdir -p $out/${path} - ${srcList path (map (s: "${s}") srcs)} - ${asmBuild} - ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} - ${asmPack} - '') // { goDeps = uniqueDeps; goImportPath = path; }; - - # Build a tree of Go libraries out of an external Go source - # directory that follows the standard Go layout and was not built - # with buildGo.nix. - # - # The derivation for each actual package will reside in an attribute - # named "gopkg", and an attribute named "gobin" for binaries. - external = import ./external { inherit pkgs program package; }; - - # Import support libraries needed for protobuf & gRPC support - protoLibs = import ./proto.nix { - inherit external; - }; - - # Build a Go library out of the specified protobuf definition. - proto = { name, proto, path ? name, extraDeps ? [] }: (makeOverridable package) { - inherit name path; - deps = [ protoLibs.goProto.proto.gopkg ] ++ extraDeps; - srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} '' - cp ${proto} ${baseNameOf proto} - ${protobuf}/bin/protoc --plugin=${protoLibs.goProto.protoc-gen-go.gopkg}/bin/protoc-gen-go \ - --go_out=plugins=grpc,import_path=${baseNameOf path}:. ${baseNameOf proto} - mv *.pb.go $out - ''); - }; - - # Build a Go library out of the specified gRPC definition. - grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc.gopkg ]; }); - -in { - # Only the high-level builder functions are exposed, but made - # overrideable. - program = makeOverridable program; - package = makeOverridable package; - proto = makeOverridable proto; - grpc = makeOverridable grpc; - external = makeOverridable external; -} diff --git a/overrides/buildGo/example/default.nix b/overrides/buildGo/example/default.nix deleted file mode 100644 index 5abed1fbbcb5..000000000000 --- a/overrides/buildGo/example/default.nix +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2019 Google LLC. -# SPDX-License-Identifier: Apache-2.0 - -# This file provides examples for how to use the various builder -# functions provided by `buildGo`. -# -# The features used in the example are not exhaustive, but should give -# users a quick introduction to how to use buildGo. - -let - buildGo = import ../buildGo.nix {}; - - # Example use of buildGo.package, which creates an importable Go - # package from the specified source files. - examplePackage = buildGo.package { - name = "example"; - srcs = [ - ./lib.go - ]; - }; - - # Example use of buildGo.proto, which generates a Go library from a - # Protobuf definition file. - exampleProto = buildGo.proto { - name = "exampleproto"; - proto = ./thing.proto; - }; - - # Example use of buildGo.program, which builds an executable using - # the specified name and dependencies (which in turn must have been - # created via buildGo.package etc.) -in buildGo.program { - name = "example"; - - srcs = [ - ./main.go - ]; - - deps = [ - examplePackage - exampleProto - ]; - - x_defs = { - "main.Flag" = "successfully"; - }; -} diff --git a/overrides/buildGo/example/lib.go b/overrides/buildGo/example/lib.go deleted file mode 100644 index 8a61370e994c..000000000000 --- a/overrides/buildGo/example/lib.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2019 Google LLC. -// SPDX-License-Identifier: Apache-2.0 - -package example - -// UUID returns a totally random, carefully chosen UUID -func UUID() string { - return "3640932f-ad40-4bc9-b45d-f504a0f5910a" -} diff --git a/overrides/buildGo/example/main.go b/overrides/buildGo/example/main.go deleted file mode 100644 index bbcedbff8726..000000000000 --- a/overrides/buildGo/example/main.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2019 Google LLC. -// SPDX-License-Identifier: Apache-2.0 -// -// Package main provides a tiny example program for the Bazel-style -// Nix build system for Go. - -package main - -import ( - "example" - "exampleproto" - "fmt" -) - -var Flag string = "unsuccessfully" - -func main() { - thing := exampleproto.Thing{ - Id: example.UUID(), - KindOfThing: "test thing", - } - - fmt.Printf("The thing is a %s with ID %q\n", thing.Id, thing.KindOfThing) - fmt.Printf("The flag has been %s set\n", Flag) -} diff --git a/overrides/buildGo/example/thing.proto b/overrides/buildGo/example/thing.proto deleted file mode 100644 index 0cb34124dfb9..000000000000 --- a/overrides/buildGo/example/thing.proto +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2019 Google LLC. -// SPDX-License-Identifier: Apache-2.0 - -syntax = "proto3"; -package exampleProto; - -message Thing { - string id = 1; - string kind_of_thing = 2; -} diff --git a/overrides/buildGo/external/default.nix b/overrides/buildGo/external/default.nix deleted file mode 100644 index 48f678688eec..000000000000 --- a/overrides/buildGo/external/default.nix +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2019 Google LLC. -# SPDX-License-Identifier: Apache-2.0 -{ pkgs, program, package }: - -let - inherit (builtins) - elemAt - foldl' - fromJSON - head - length - listToAttrs - readFile - replaceStrings - tail - throw; - - inherit (pkgs) lib runCommand go jq ripgrep; - - pathToName = p: replaceStrings ["/"] ["_"] (toString p); - - # Collect all non-vendored dependencies from the Go standard library - # into a file that can be used to filter them out when processing - # dependencies. - stdlibPackages = runCommand "stdlib-pkgs.json" {} '' - export HOME=$PWD - export GOPATH=/dev/null - ${go}/bin/go list all | \ - ${ripgrep}/bin/rg -v 'vendor' | \ - ${jq}/bin/jq -R '.' | \ - ${jq}/bin/jq -c -s 'map({key: ., value: true}) | from_entries' \ - > $out - ''; - - analyser = program { - name = "analyser"; - - srcs = [ - ./main.go - ]; - - x_defs = { - "main.stdlibList" = "${stdlibPackages}"; - }; - }; - - mkset = path: value: - if path == [] then { gopkg = value; } - else { "${head path}" = mkset (tail path) value; }; - - last = l: elemAt l ((length l) - 1); - - toPackage = self: src: path: depMap: entry: - let - localDeps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) ( - throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'" - ) self) entry.localDeps; - - foreignDeps = map (d: lib.attrByPath [ d ] ( - throw "missing foreign dependency '${d}' in '${path}'" - ) depMap) entry.foreignDeps; - - args = { - srcs = map (f: src + ("/" + f)) entry.files; - deps = localDeps ++ foreignDeps; - }; - - libArgs = args // { - name = pathToName entry.name; - path = lib.concatStringsSep "/" ([ path ] ++ entry.locator); - sfiles = map (f: src + ("/" + f)) entry.sfiles; - }; - - binArgs = args // { - name = (last ((lib.splitString "/" path) ++ entry.locator)); - }; - in if entry.isCommand then (program binArgs) else (package libArgs); - -in { src, path, deps ? [] }: let - # Build a map of dependencies (from their import paths to their - # derivation) so that they can be conditionally imported only in - # sub-packages that require them. - depMap = listToAttrs (map (d: { - name = d.goImportPath; - value = d; - }) deps); - - name = pathToName path; - analysisOutput = runCommand "${name}-structure.json" {} '' - ${analyser}/bin/analyser -path ${path} -source ${src} > $out - ''; - analysis = fromJSON (readFile analysisOutput); -in lib.fix(self: foldl' lib.recursiveUpdate {} ( - map (entry: mkset entry.locator (toPackage self src path depMap entry)) analysis -)) diff --git a/overrides/buildGo/external/main.go b/overrides/buildGo/external/main.go deleted file mode 100644 index aa4a813d32bd..000000000000 --- a/overrides/buildGo/external/main.go +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2019 Google LLC. -// SPDX-License-Identifier: Apache-2.0 - -// This tool analyses external (i.e. not built with `buildGo.nix`) Go -// packages to determine a build plan that Nix can import. -package main - -import ( - "encoding/json" - "flag" - "fmt" - "go/build" - "io/ioutil" - "log" - "os" - "path" - "path/filepath" - "strings" -) - -// Path to a JSON file describing all standard library import paths. -// This file is generated and set here by Nix during the build -// process. -var stdlibList string - -// pkg describes a single Go package within the specified source -// directory. -// -// Return information includes the local (relative from project root) -// and external (none-stdlib) dependencies of this package. -type pkg struct { - Name string `json:"name"` - Locator []string `json:"locator"` - Files []string `json:"files"` - SFiles []string `json:"sfiles"` - LocalDeps [][]string `json:"localDeps"` - ForeignDeps []string `json:"foreignDeps"` - IsCommand bool `json:"isCommand"` -} - -// findGoDirs returns a filepath.WalkFunc that identifies all -// directories that contain Go source code in a certain tree. -func findGoDirs(at string) ([]string, error) { - dirSet := make(map[string]bool) - - err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error { - name := info.Name() - // Skip folders that are guaranteed to not be relevant - if info.IsDir() && (name == "testdata" || name == ".git") { - return filepath.SkipDir - } - - // If the current file is a Go file, then the directory is popped - // (i.e. marked as a Go directory). - if !info.IsDir() && strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go") { - dirSet[filepath.Dir(path)] = true - } - - return nil - }) - - if err != nil { - return nil, err - } - - goDirs := []string{} - for k, _ := range dirSet { - goDirs = append(goDirs, k) - } - - return goDirs, nil -} - -// analysePackage loads and analyses the imports of a single Go -// package, returning the data that is required by the Nix code to -// generate a derivation for this package. -func analysePackage(root, source, importpath string, stdlib map[string]bool) (pkg, error) { - ctx := build.Default - ctx.CgoEnabled = false - - p, err := ctx.ImportDir(source, build.IgnoreVendor) - if err != nil { - return pkg{}, err - } - - local := [][]string{} - foreign := []string{} - - for _, i := range p.Imports { - if stdlib[i] { - continue - } - - if i == importpath { - local = append(local, []string{}) - } else if strings.HasPrefix(i, importpath) { - local = append(local, strings.Split(strings.TrimPrefix(i, importpath+"/"), "/")) - } else { - foreign = append(foreign, i) - } - } - - prefix := strings.TrimPrefix(source, root+"/") - - locator := []string{} - if len(prefix) != len(source) { - locator = strings.Split(prefix, "/") - } else { - // Otherwise, the locator is empty since its the root package and - // no prefix should be added to files. - prefix = "" - } - - files := []string{} - for _, f := range p.GoFiles { - files = append(files, path.Join(prefix, f)) - } - - sfiles := []string{} - for _, f := range p.SFiles { - sfiles = append(sfiles, path.Join(prefix, f)) - } - - return pkg{ - Name: path.Join(importpath, prefix), - Locator: locator, - Files: files, - SFiles: sfiles, - LocalDeps: local, - ForeignDeps: foreign, - IsCommand: p.IsCommand(), - }, nil -} - -func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) { - f, err := ioutil.ReadFile(from) - if err != nil { - return - } - - err = json.Unmarshal(f, &pkgs) - return -} - -func main() { - source := flag.String("source", "", "path to directory with sources to process") - path := flag.String("path", "", "import path for the package") - - flag.Parse() - - if *source == "" { - log.Fatalf("-source flag must be specified") - } - - stdlibPkgs, err := loadStdlibPkgs(stdlibList) - if err != nil { - log.Fatalf("failed to load standard library index from %q: %s\n", stdlibList, err) - } - - goDirs, err := findGoDirs(*source) - if err != nil { - log.Fatalf("failed to walk source directory '%s': %s\n", source, err) - } - - all := []pkg{} - for _, d := range goDirs { - analysed, err := analysePackage(*source, d, *path, stdlibPkgs) - - // If the Go source analysis returned "no buildable Go files", - // that directory should be skipped. - // - // This might be due to `+build` flags on the platform and other - // reasons (such as test files). - if _, ok := err.(*build.NoGoError); ok { - continue - } - - if err != nil { - log.Fatalf("failed to analyse package at %q: %s", d, err) - } - all = append(all, analysed) - } - - j, _ := json.Marshal(all) - fmt.Println(string(j)) -} diff --git a/overrides/buildGo/proto.nix b/overrides/buildGo/proto.nix deleted file mode 100644 index 2ece948ebd84..000000000000 --- a/overrides/buildGo/proto.nix +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2019 Google LLC. -# SPDX-License-Identifier: Apache-2.0 -# -# This file provides derivations for the dependencies of a gRPC -# service in Go. - -{ external }: - -let - inherit (builtins) fetchGit map; -in rec { - goProto = external { - path = "github.com/golang/protobuf"; - src = fetchGit { - url = "https://github.com/golang/protobuf"; - rev = "ed6926b37a637426117ccab59282c3839528a700"; - }; - }; - - xnet = external { - path = "golang.org/x/net"; - - src = fetchGit { - url = "https://go.googlesource.com/net"; - rev = "ffdde105785063a81acd95bdf89ea53f6e0aac2d"; - }; - - deps = map (p: p.gopkg) [ - xtext.secure.bidirule - xtext.unicode.bidi - xtext.unicode.norm - ]; - }; - - xsys = external { - path = "golang.org/x/sys"; - src = fetchGit { - url = "https://go.googlesource.com/sys"; - rev = "bd437916bb0eb726b873ee8e9b2dcf212d32e2fd"; - }; - }; - - xtext = external { - path = "golang.org/x/text"; - src = fetchGit { - url = "https://go.googlesource.com/text"; - rev = "cbf43d21aaebfdfeb81d91a5f444d13a3046e686"; - }; - }; - - genproto = external { - path = "google.golang.org/genproto"; - src = fetchGit { - url = "https://github.com/google/go-genproto"; - rev = "83cc0476cb11ea0da33dacd4c6354ab192de6fe6"; - }; - - deps = with goProto; map (p: p.gopkg) [ - proto - ptypes.any - ]; - }; - - goGrpc = external { - path = "google.golang.org/grpc"; - deps = map (p: p.gopkg) ([ - xnet.trace - xnet.http2 - xsys.unix - xnet.http2.hpack - genproto.googleapis.rpc.status - ] ++ (with goProto; [ - proto - ptypes - ptypes.duration - ptypes.timestamp - ])); - - src = fetchGit { - url = "https://github.com/grpc/grpc-go"; - rev = "d8e3da36ac481ef00e510ca119f6b68177713689"; - }; - }; -} |