-- cgit 1.4.1 From 24225580e8dd6dbbb930e01d0b4947f2feaa70c7 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sat, 23 Nov 2019 15:25:11 +0000 Subject: feat: Check in buildGo with program & package builders --- buildGo.nix | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 buildGo.nix diff --git a/buildGo.nix b/buildGo.nix new file mode 100644 index 000000000000..090627c591a3 --- /dev/null +++ b/buildGo.nix @@ -0,0 +1,70 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 +# +# buildGo provides Nix functions to build Go packages in the style of Bazel's +# rules_go. + +{ pkgs ? import {} +, ... }: + +let + inherit (builtins) + attrNames + baseNameOf + elemAt + filter + map + match + readDir; + + inherit (pkgs) lib go runCommand; + + # Helpers for low-level Go compiler invocations + spaceOut = lib.concatStringsSep " "; + + includeDepSrc = dep: "-I ${dep}"; + includeSources = deps: spaceOut (map includeDepSrc deps); + + includeDepLib = dep: "-L ${dep}"; + includeLibs = deps: spaceOut (map includeDepLib deps); + + srcBasename = src: elemAt (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src)) 1; + srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}"; + srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs); + + isGoFile = f: (match ".*\.go" f) != null; + isGoTest = f: (match ".*_test\.go" f) != null; + goFileFilter = k: v: (v == "regular") && (isGoFile k) && (!isGoTest k); + goFilesIn = dir: + let files = readDir dir; + goFiles = filter (f: goFileFilter f files."${f}") (attrNames files); + in map (f: dir + "/" + f) goFiles; + + allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps))); + + # High-level build functions + + # Build a Go program out of the specified files and dependencies. + program = { name, srcs, deps ? [] }: + let uniqueDeps = allDeps deps; + in runCommand name {} '' + ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs} + mkdir -p $out/bin + ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${includeLibs uniqueDeps} ${name}.a + ''; + + # Build a Go library assembled out of the specified files. + # + # This outputs both the sources and compiled binary, as both are + # needed when downstream packages depend on it. + package = { name, srcs, deps ? [], path ? name }: + let uniqueDeps = allDeps deps; + in (runCommand "golib-${name}" {} '' + mkdir -p $out/${path} + ${srcList path (map (s: "${s}") srcs)} + ${go}/bin/go tool compile -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} + '') // { goDeps = uniqueDeps; }; +in { + # Only the high-level builder functions are exposed + inherit program package; +} -- cgit 1.4.1 From d441e035aabac877948f754289b3ab028408550b Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 24 Nov 2019 20:17:17 +0000 Subject: feat: Add support for Protobuf-generated libraries Adds a 'buildGo.proto' function which takes a single .proto file as its source and generates a corresponding Go library which can then be imported. 'proto' takes these arguments (Yants-style type definition): struct "protoArgs" { # required: name = string; proto = path; # optional: extraDeps = list goLib; # defaults to [ ] protocFlags = option string; } Note that proto libraries will automatically have dependencies for the required protobuf Go libraries added to them. gRPC is not (yet) supported. --- buildGo.nix | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 75 insertions(+), 3 deletions(-) diff --git a/buildGo.nix b/buildGo.nix index 090627c591a3..029271f2f7d4 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -15,9 +15,11 @@ let filter map match - readDir; + readDir + replaceStrings + toPath; - inherit (pkgs) lib go runCommand; + inherit (pkgs) lib go runCommand fetchFromGitHub protobuf; # Helpers for low-level Go compiler invocations spaceOut = lib.concatStringsSep " "; @@ -64,7 +66,77 @@ let ${srcList path (map (s: "${s}") srcs)} ${go}/bin/go tool compile -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} '') // { goDeps = uniqueDeps; }; + + # Build a Go library out of the specified protobuf definition. + proto = { name, proto, path ? name, protocFlags ? "", extraDeps ? [] }: package { + inherit name path; + deps = [ goProto ] ++ extraDeps; + srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} '' + cp ${proto} ${baseNameOf proto} + ${protobuf}/bin/protoc --plugin=${protocGo}/bin/protoc-gen-go \ + --go_out=${protocFlags}import_path=${baseNameOf path}:. ${baseNameOf proto} + mv *.pb.go $out + ''); + }; + + # Protobuf & gRPC integration requires these dependencies: + proto-go-src = fetchFromGitHub { + owner = "golang"; + repo = "protobuf"; + rev = "ed6926b37a637426117ccab59282c3839528a700"; + sha256 = "0fynqrim022x9xi2bivkw19npbz4316v4yr7mb677s9s36z4dc4h"; + }; + + protoPart = path: deps: package { + inherit deps; + name = replaceStrings ["/"] ["_"] path; + path = "github.com/golang/protobuf/${path}"; + srcs = goFilesIn (toPath "${proto-go-src}/${path}"); + }; + + goProto = + let + protobuf = package { + name = "protobuf"; + path = "github.com/golang/protobuf/proto"; + # TODO(tazjin): How does this build toggle work? + srcs = filter + (f: (match "(.*)/pointer_reflect.go" f) == null) + (goFilesIn (toPath "${proto-go-src}/proto")); + }; + type = name: protoPart "ptypes/${name}" [ protobuf ]; + descriptor = protoPart "descriptor" [ protobuf ]; + ptypes = package { + name = "ptypes"; + path = "github.com/golang/protobuf/ptypes"; + srcs = goFilesIn (toPath "${proto-go-src}/ptypes"); + deps = map type [ + "any" + "duration" + "empty" + "struct" + "timestamp" + "wrappers" + ]; + }; + in protobuf // { goDeps = allDeps (protobuf.goDeps ++ [ ptypes ]); }; + + protocDescriptor = (protoPart "protoc-gen-go/descriptor" [ goProto ]); + protocGo = + let + generator = protoPart "protoc-gen-go/generator" [ + (protoPart "protoc-gen-go/generator/internal/remap" []) + (protoPart "protoc-gen-go/plugin" [ protocDescriptor ]) + ]; + grpc = protoPart "protoc-gen-go/grpc" [ generator ]; + in program { + name = "protoc-gen-go"; + deps = [ goProto grpc generator ]; + srcs = filter + (f: (match "(.*)/doc.go" f) == null) + (goFilesIn (toPath "${proto-go-src}/protoc-gen-go")); + }; in { # Only the high-level builder functions are exposed - inherit program package; + inherit program package proto; } -- cgit 1.4.1 From 071babf14824aee035dd6c3ddff1957662aa7c98 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 24 Nov 2019 20:34:30 +0000 Subject: feat(example): Add an example for how to use buildGo builders --- example/default.nix | 43 +++++++++++++++++++++++++++++++++++++++++++ example/lib.go | 9 +++++++++ example/main.go | 22 ++++++++++++++++++++++ example/thing.proto | 10 ++++++++++ 4 files changed, 84 insertions(+) create mode 100644 example/default.nix create mode 100644 example/lib.go create mode 100644 example/main.go create mode 100644 example/thing.proto diff --git a/example/default.nix b/example/default.nix new file mode 100644 index 000000000000..407f55079359 --- /dev/null +++ b/example/default.nix @@ -0,0 +1,43 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 + +# This file provides examples for how to use the various builder +# functions provided by `buildGo`. +# +# The features used in the example are not exhaustive, but should give +# users a quick introduction to how to use buildGo. + +let + buildGo = import ../buildGo.nix {}; + + # Example use of buildGo.package, which creates an importable Go + # package from the specified source files. + examplePackage = buildGo.package { + name = "example"; + srcs = [ + ./lib.go + ]; + }; + + # Example use of buildGo.proto, which generates a Go library from a + # Protobuf definition file. + exampleProto = buildGo.proto { + name = "exampleproto"; + proto = ./thing.proto; + }; + + # Example use of buildGo.program, which builds an executable using + # the specified name and dependencies (which in turn must have been + # created via buildGo.package etc.) +in buildGo.program { + name = "example"; + + srcs = [ + ./main.go + ]; + + deps = [ + examplePackage + exampleProto + ]; +} diff --git a/example/lib.go b/example/lib.go new file mode 100644 index 000000000000..8a61370e994c --- /dev/null +++ b/example/lib.go @@ -0,0 +1,9 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 + +package example + +// UUID returns a totally random, carefully chosen UUID +func UUID() string { + return "3640932f-ad40-4bc9-b45d-f504a0f5910a" +} diff --git a/example/main.go b/example/main.go new file mode 100644 index 000000000000..1db1bb9e23e9 --- /dev/null +++ b/example/main.go @@ -0,0 +1,22 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 +// +// Package main provides a tiny example program for the Bazel-style +// Nix build system for Go. + +package main + +import ( + "example" + "exampleproto" + "fmt" +) + +func main() { + thing := exampleproto.Thing{ + Id: example.UUID(), + KindOfThing: "test thing", + } + + fmt.Printf("The thing is a %s with ID %q\n", thing.Id, thing.KindOfThing) +} diff --git a/example/thing.proto b/example/thing.proto new file mode 100644 index 000000000000..0cb34124dfb9 --- /dev/null +++ b/example/thing.proto @@ -0,0 +1,10 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; +package exampleProto; + +message Thing { + string id = 1; + string kind_of_thing = 2; +} -- cgit 1.4.1 From 0fd74669229415907353563b3dbc2847d8a80942 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 24 Nov 2019 20:41:01 +0000 Subject: feat(buildGo): Add support for x_defs option in buildGo.program This lets users define an attribute set with link time options. --- buildGo.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/buildGo.nix b/buildGo.nix index 029271f2f7d4..dce5502658c5 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -44,15 +44,17 @@ let allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps))); + xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); + # High-level build functions # Build a Go program out of the specified files and dependencies. - program = { name, srcs, deps ? [] }: + program = { name, srcs, deps ? [], x_defs ? {} }: let uniqueDeps = allDeps deps; in runCommand name {} '' ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs} mkdir -p $out/bin - ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${includeLibs uniqueDeps} ${name}.a + ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a ''; # Build a Go library assembled out of the specified files. -- cgit 1.4.1 From 473421dbb81fedbec7f1f6e788c813b5fe9f4604 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 24 Nov 2019 20:41:29 +0000 Subject: feat(example): Demonstrate usage of x_defs flag to buildGo.program --- example/default.nix | 4 ++++ example/main.go | 3 +++ 2 files changed, 7 insertions(+) diff --git a/example/default.nix b/example/default.nix index 407f55079359..5abed1fbbcb5 100644 --- a/example/default.nix +++ b/example/default.nix @@ -40,4 +40,8 @@ in buildGo.program { examplePackage exampleProto ]; + + x_defs = { + "main.Flag" = "successfully"; + }; } diff --git a/example/main.go b/example/main.go index 1db1bb9e23e9..bbcedbff8726 100644 --- a/example/main.go +++ b/example/main.go @@ -12,6 +12,8 @@ import ( "fmt" ) +var Flag string = "unsuccessfully" + func main() { thing := exampleproto.Thing{ Id: example.UUID(), @@ -19,4 +21,5 @@ func main() { } fmt.Printf("The thing is a %s with ID %q\n", thing.Id, thing.KindOfThing) + fmt.Printf("The flag has been %s set\n", Flag) } -- cgit 1.4.1 From 7d26550c11e6fdde5b3a0a052b917aef16dc9816 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Mon, 25 Nov 2019 16:42:49 +0000 Subject: feat(buildGo): Add support for building "external" Go libraries Adds a buildGo.external function that can build packages following the default go-tool package layout. Dependencies work the same way as they do for other buildGo-packages, but instead of being passed straight to the compiler a fake GOPATH is assembled using a symlink forest. External currently supports very few direct configuration options and was primarily created to build the protobuf packages, but it is also useful for including external dependencies in buildGo-native projects. The previous complex build logic for the protobuf package has been replaced with a call to `external`. --- buildGo.nix | 104 +++++++++++++++++++++++++++++------------------------------- 1 file changed, 51 insertions(+), 53 deletions(-) diff --git a/buildGo.nix b/buildGo.nix index dce5502658c5..1a7ed66c7ecc 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -11,15 +11,15 @@ let inherit (builtins) attrNames baseNameOf + dirOf elemAt filter map match readDir - replaceStrings - toPath; + replaceStrings; - inherit (pkgs) lib go runCommand fetchFromGitHub protobuf; + inherit (pkgs) lib go runCommand fetchFromGitHub protobuf symlinkJoin; # Helpers for low-level Go compiler invocations spaceOut = lib.concatStringsSep " "; @@ -46,6 +46,8 @@ let xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); + pathToName = p: replaceStrings ["/"] ["_"] (toString p); + # High-level build functions # Build a Go program out of the specified files and dependencies. @@ -75,12 +77,52 @@ let deps = [ goProto ] ++ extraDeps; srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} '' cp ${proto} ${baseNameOf proto} - ${protobuf}/bin/protoc --plugin=${protocGo}/bin/protoc-gen-go \ + ${protobuf}/bin/protoc --plugin=${goProto}/bin/protoc-gen-go \ --go_out=${protocFlags}import_path=${baseNameOf path}:. ${baseNameOf proto} mv *.pb.go $out ''); }; + # Build an externally defined Go library using `go build` itself. + # + # Libraries built this way can be included in any standard buildGo + # build. + # + # Contrary to other functions, `src` is expected to point at a + # single directory containing the root of the external library. + external = { path, src, deps ? [] }: + let + name = pathToName path; + uniqueDeps = allDeps deps; + srcDir = runCommand "goext-src-${name}" {} '' + mkdir -p $out/${dirOf path} + cp -r ${src} $out/${dirOf path}/${baseNameOf path} + ''; + gopathSrc = symlinkJoin { + name = "gopath-${name}"; + paths = uniqueDeps ++ [ srcDir ]; + }; + gopathPkg = runCommand "goext-pkg-${name}" {} '' + mkdir -p gopath $out + export GOPATH=$PWD/gopath + ln -s ${gopathSrc} gopath/src + ${go}/bin/go install ${path}/... + + if [[ -d gopath/pkg/linux_amd64 ]]; then + echo "Installing Go packages for ${path}" + mv gopath/pkg/linux_amd64/* $out + fi + + if [[ -d gopath/bin ]]; then + echo "Installing Go binaries for ${path}" + mv gopath/bin $out/bin + fi + ''; + in symlinkJoin { + name = "goext-${name}"; + paths = [ gopathSrc gopathPkg ]; + } // { goDeps = uniqueDeps; }; + # Protobuf & gRPC integration requires these dependencies: proto-go-src = fetchFromGitHub { owner = "golang"; @@ -89,56 +131,12 @@ let sha256 = "0fynqrim022x9xi2bivkw19npbz4316v4yr7mb677s9s36z4dc4h"; }; - protoPart = path: deps: package { - inherit deps; - name = replaceStrings ["/"] ["_"] path; - path = "github.com/golang/protobuf/${path}"; - srcs = goFilesIn (toPath "${proto-go-src}/${path}"); + goProto = external { + path = "github.com/golang/protobuf"; + src = proto-go-src; + deps = []; }; - - goProto = - let - protobuf = package { - name = "protobuf"; - path = "github.com/golang/protobuf/proto"; - # TODO(tazjin): How does this build toggle work? - srcs = filter - (f: (match "(.*)/pointer_reflect.go" f) == null) - (goFilesIn (toPath "${proto-go-src}/proto")); - }; - type = name: protoPart "ptypes/${name}" [ protobuf ]; - descriptor = protoPart "descriptor" [ protobuf ]; - ptypes = package { - name = "ptypes"; - path = "github.com/golang/protobuf/ptypes"; - srcs = goFilesIn (toPath "${proto-go-src}/ptypes"); - deps = map type [ - "any" - "duration" - "empty" - "struct" - "timestamp" - "wrappers" - ]; - }; - in protobuf // { goDeps = allDeps (protobuf.goDeps ++ [ ptypes ]); }; - - protocDescriptor = (protoPart "protoc-gen-go/descriptor" [ goProto ]); - protocGo = - let - generator = protoPart "protoc-gen-go/generator" [ - (protoPart "protoc-gen-go/generator/internal/remap" []) - (protoPart "protoc-gen-go/plugin" [ protocDescriptor ]) - ]; - grpc = protoPart "protoc-gen-go/grpc" [ generator ]; - in program { - name = "protoc-gen-go"; - deps = [ goProto grpc generator ]; - srcs = filter - (f: (match "(.*)/doc.go" f) == null) - (goFilesIn (toPath "${proto-go-src}/protoc-gen-go")); - }; in { # Only the high-level builder functions are exposed - inherit program package proto; + inherit program package proto external; } -- cgit 1.4.1 From 8b6b08b814af72e8b2f6281037f089e099a41e25 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Tue, 26 Nov 2019 12:11:43 +0000 Subject: feat(buildGo): Add 'srcOnly' and 'targets' parameters for external Adds two new parameters to buildGo.external: * `srcOnly` toggles whether the created derivation should contain only the source code, or the built package. This is useful in situations where some sub-packages of a larger package are needed and the build should be deferred to the package depending on them. It defaults to false, meaning that external packages are built by default. * `targets` controls which "sub-packages" of the target package are built. It defaults to building all sub-packages. --- buildGo.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/buildGo.nix b/buildGo.nix index 1a7ed66c7ecc..12504ce57103 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -90,7 +90,7 @@ let # # Contrary to other functions, `src` is expected to point at a # single directory containing the root of the external library. - external = { path, src, deps ? [] }: + external = { path, src, deps ? [], srcOnly ? false, targets ? [ "..." ] }: let name = pathToName path; uniqueDeps = allDeps deps; @@ -106,7 +106,7 @@ let mkdir -p gopath $out export GOPATH=$PWD/gopath ln -s ${gopathSrc} gopath/src - ${go}/bin/go install ${path}/... + ${go}/bin/go install ${spaceOut (map (t: path + "/" + t) targets)} if [[ -d gopath/pkg/linux_amd64 ]]; then echo "Installing Go packages for ${path}" @@ -118,10 +118,10 @@ let mv gopath/bin $out/bin fi ''; - in symlinkJoin { + in (if srcOnly then gopathSrc else symlinkJoin { name = "goext-${name}"; paths = [ gopathSrc gopathPkg ]; - } // { goDeps = uniqueDeps; }; + }) // { goDeps = uniqueDeps; }; # Protobuf & gRPC integration requires these dependencies: proto-go-src = fetchFromGitHub { -- cgit 1.4.1 From d3e8774a8e159984bfa2864d383a1d9b641a4c6f Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Tue, 26 Nov 2019 12:15:18 +0000 Subject: feat(proto): Add protobuf & gRPC dependencies via external Moves the Protobuf & gRPC dependencies to a separate file which uses buildGo.external to build the dependencies. The versions are pinned at master of 2019-11-26. --- buildGo.nix | 14 ++---------- proto.nix | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 12 deletions(-) create mode 100644 proto.nix diff --git a/buildGo.nix b/buildGo.nix index 12504ce57103..bf8625c1b440 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -123,18 +123,8 @@ let paths = [ gopathSrc gopathPkg ]; }) // { goDeps = uniqueDeps; }; - # Protobuf & gRPC integration requires these dependencies: - proto-go-src = fetchFromGitHub { - owner = "golang"; - repo = "protobuf"; - rev = "ed6926b37a637426117ccab59282c3839528a700"; - sha256 = "0fynqrim022x9xi2bivkw19npbz4316v4yr7mb677s9s36z4dc4h"; - }; - - goProto = external { - path = "github.com/golang/protobuf"; - src = proto-go-src; - deps = []; + protoLibs = import ./proto.nix { + inherit external; }; in { # Only the high-level builder functions are exposed diff --git a/proto.nix b/proto.nix new file mode 100644 index 000000000000..e773af031873 --- /dev/null +++ b/proto.nix @@ -0,0 +1,72 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 +# +# This file provides derivations for the dependencies of a gRPC +# service in Go. + +{ external }: + +let + inherit (builtins) fetchGit map; +in rec { + goProto = external { + path = "github.com/golang/protobuf"; + src = fetchGit { + url = "https://github.com/golang/protobuf"; + rev = "ed6926b37a637426117ccab59282c3839528a700"; + }; + }; + + xnet = external { + path = "golang.org/x/net"; + srcOnly = true; + deps = [ xtext ]; + src = fetchGit { + url = "https://go.googlesource.com/net"; + rev = "ffdde105785063a81acd95bdf89ea53f6e0aac2d"; + }; + }; + + xsys = external { + path = "golang.org/x/sys"; + srcOnly = true; + src = fetchGit { + url = "https://go.googlesource.com/sys"; + rev = "bd437916bb0eb726b873ee8e9b2dcf212d32e2fd"; + }; + }; + + xtext = external { + path = "golang.org/x/text"; + srcOnly = true; + src = fetchGit { + url = "https://go.googlesource.com/text"; + rev = "cbf43d21aaebfdfeb81d91a5f444d13a3046e686"; + }; + }; + + genproto = external { + path = "google.golang.org/genproto"; + srcOnly = true; + src = fetchGit { + url = "https://github.com/google/go-genproto"; + rev = "83cc0476cb11ea0da33dacd4c6354ab192de6fe6"; + }; + }; + + goGrpc = external { + path = "google.golang.org/grpc"; + deps = [ goProto xnet xsys genproto ]; + + src = fetchGit { + url = "https://github.com/grpc/grpc-go"; + rev = "d8e3da36ac481ef00e510ca119f6b68177713689"; + }; + + targets = [ + "." + "codes" + "status" + ]; + }; +} -- cgit 1.4.1 From 9280cf97151fd14269342310ad1acfb6ccd37369 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Tue, 26 Nov 2019 12:16:27 +0000 Subject: feat(buildGo): Add support for gRPC packages Introduces buildGo.grpc which is like buildGo.proto, but adds dependencies on the gRPC libraries. --- buildGo.nix | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/buildGo.nix b/buildGo.nix index bf8625c1b440..fba4e18e9bce 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -72,17 +72,20 @@ let '') // { goDeps = uniqueDeps; }; # Build a Go library out of the specified protobuf definition. - proto = { name, proto, path ? name, protocFlags ? "", extraDeps ? [] }: package { + proto = { name, proto, path ? name, extraDeps ? [] }: package { inherit name path; - deps = [ goProto ] ++ extraDeps; + deps = [ protoLibs.goProto ] ++ extraDeps; srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} '' cp ${proto} ${baseNameOf proto} - ${protobuf}/bin/protoc --plugin=${goProto}/bin/protoc-gen-go \ - --go_out=${protocFlags}import_path=${baseNameOf path}:. ${baseNameOf proto} + ${protobuf}/bin/protoc --plugin=${protoLibs.goProto}/bin/protoc-gen-go \ + --go_out=plugins=grpc,import_path=${baseNameOf path}:. ${baseNameOf proto} mv *.pb.go $out ''); }; + # Build a Go library out of the specified gRPC definition. + grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc ]; }); + # Build an externally defined Go library using `go build` itself. # # Libraries built this way can be included in any standard buildGo @@ -128,5 +131,5 @@ let }; in { # Only the high-level builder functions are exposed - inherit program package proto external; + inherit program package proto grpc external; } -- cgit 1.4.1 From 28e587b348a8aaa7af00a004c05286af9d35ca9a Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Wed, 27 Nov 2019 15:14:30 +0000 Subject: docs: Add README file that describes project usage & background --- README.md | 139 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000000..489cf128173f --- /dev/null +++ b/README.md @@ -0,0 +1,139 @@ +buildGo.nix +=========== + +This is an alternative [Nix][] build system for [Go][]. It supports building Go +libraries and programs, and even automatically generating Protobuf & gRPC +libraries. + +*Note:* This will probably end up being folded into [Nixery][]. + +## Background + +Most language-specific Nix tooling outsources the build to existing +language-specific build tooling, which essentially means that Nix ends up being +a wrapper around all sorts of external build systems. + +However, systems like [Bazel][] take an alternative approach in which the +compiler is invoked directly and the composition of programs and libraries stays +within a single homogeneous build system. + +Users don't need to learn per-language build systems and especially for +companies with large monorepo-setups ([like Google][]) this has huge +productivity impact. + +This project is an attempt to prove that Nix can be used in a similar style to +build software directly, rather than shelling out to other build systems. + +## Example + +Given a program layout like this: + +``` +. +├── lib <-- some library component +│   ├── bar.go +│   └── foo.go +├── api.proto <-- gRPC API definition +├── main.go <-- program implementation +└── default.nix <-- build instructions +``` + +The contents of `default.nix` could look like this: + +```nix +{ buildGo }: + +let + api = buildGo.grpc { + name = "someapi"; + proto = ./api.proto; + }; + + lib = buildGo.package { + name = "somelib"; + srcs = [ + ./lib/bar.go + ./lib/foo.go + ]; + }; +in buildGo.program { + name = "my-program"; + deps = [ api lib ]; + + srcs = [ + ./main.go + ]; +} +``` + +(If you don't know how to read Nix, check out [nix-1p][]) + +## Usage + +`buildGo` exposes five different functions: + +* `buildGo.program`: Build a Go binary out of the specified source files. + + | parameter | type | use | required? | + |-----------|-------------------------|------------------------------------------------|-----------| + | `name` | `string` | Name of the program (and resulting executable) | yes | + | `srcs` | `list` | List of paths to source files | yes | + | `deps` | `list` | List of dependencies (i.e. other Go libraries) | no | + | `x_defs` | `attrs` | Attribute set of linker vars (i.e. `-X`-flags) | no | + +* `buildGo.package`: Build a Go library out of the specified source files. + + | parameter | type | use | required? | + |-----------|--------------|------------------------------------------------|-----------| + | `name` | `string` | Name of the library (and resulting executable) | yes | + | `srcs` | `list` | List of paths to source files | yes | + | `deps` | `list` | List of dependencies (i.e. other Go libraries) | no | + | `path` | `string` | Go import path for the resulting library | no | + +* `buildGo.external`: Build a Go library or program using standard `go` tooling. + + This exists for compatibility with complex external dependencies. In theory it + is possible to write `buildGo.package` specifications for each subpackage of + an external dependency, but it is often cumbersome to do so. + + | parameter | type | use | required? | + |-----------|----------------|------------------------------------------------|-----------| + | `path` | `string` | Go import path for the resulting library | yes | + | `src` | `path` | Path to the source **directory** | yes | + | `deps` | `list` | List of dependencies (i.e. other Go libraries) | no | + | `srcOnly` | `bool` | Only copy sources, do not perform a build. | no | + | `targets` | `list` | Sub-packages to build (defaults to all) | no | + + For some examples of how `buildGo.external` is used, check out + [`proto.nix`](./proto.nix). + +* `buildGo.proto`: Build a Go library out of the specified Protobuf definition. + + | parameter | type | use | required? | + |-------------|-------------|--------------------------------------------------|-----------| + | `name` | `string` | Name for the resulting library | yes | + | `proto` | `path` | Path to the Protobuf definition file | yes | + | `path` | `string` | Import path for the resulting Go library | no | + | `extraDeps` | `list` | Additional Go dependencies to add to the library | no | + +* `buildGo.grpc`: Build a Go library out of the specified gRPC definition. + + The parameters are identical to `buildGo.proto`. + +## Current status + +This project is work-in-progress. Crucially it is lacking the following features: + +* feature flag parity with Bazel's Go rules +* documentation building +* test execution + +There are still some open questions around how to structure some of those +features in Nix. + +[Nix]: https://nixos.org/nix/ +[Go]: https://golang.org/ +[Nixery]: https://github.com/google/nixery +[Bazel]: https://bazel.build/ +[like Google]: https://ai.google/research/pubs/pub45424 +[nix-1p]: https://github.com/tazjin/nix-1p -- cgit 1.4.1 From 6a526620e25a349642b161eb687dec2e005360e5 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Mon, 9 Dec 2019 00:51:27 +0000 Subject: feat(buildGo): Add 'overrideGo' argument overriding function This makes it possible to override arguments to the Go builders downstream in the style of `overrideAttrs` from standard nixpkgs derivations. For example, given a Nix value `foo` that builds a binary called `foo` the name of this binary could be changed and a new dependency on `somelib` added like so: foo.overrideGo(old: { name = "bar"; deps = old.deps ++ [ somelib ]; }) --- buildGo.nix | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/buildGo.nix b/buildGo.nix index fba4e18e9bce..5f5118e8cec2 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -48,6 +48,13 @@ let pathToName = p: replaceStrings ["/"] ["_"] (toString p); + # Add an `overrideGo` attribute to a function result that works + # similar to `overrideAttrs`, but is used specifically for the + # arguments passed to Go builders. + makeOverridable = f: orig: (f orig) // { + overrideGo = new: makeOverridable f (orig // (new orig)); + }; + # High-level build functions # Build a Go program out of the specified files and dependencies. @@ -130,6 +137,11 @@ let inherit external; }; in { - # Only the high-level builder functions are exposed - inherit program package proto grpc external; + # Only the high-level builder functions are exposed, but made + # overrideable. + program = makeOverridable program; + package = makeOverridable package; + proto = makeOverridable proto; + grpc = makeOverridable grpc; + external = makeOverridable external; } -- cgit 1.4.1 From c40e8a4061b033282031cd1a8d64df9a591aa694 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Mon, 9 Dec 2019 00:27:05 +0000 Subject: feat(buildGo): Add new traversing external' implementation Adds an alternative implementation of a builder for external packages which traverses packages and builds up an attribute set tree out of their structure. Currently this is not functional because there is no useable method of specifying dependencies within that package set. --- buildGo.nix | 55 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/buildGo.nix b/buildGo.nix index 5f5118e8cec2..366be770a959 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -14,10 +14,12 @@ let dirOf elemAt filter + listToAttrs map match readDir - replaceStrings; + replaceStrings + toString; inherit (pkgs) lib go runCommand fetchFromGitHub protobuf symlinkJoin; @@ -34,14 +36,6 @@ let srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}"; srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs); - isGoFile = f: (match ".*\.go" f) != null; - isGoTest = f: (match ".*_test\.go" f) != null; - goFileFilter = k: v: (v == "regular") && (isGoFile k) && (!isGoTest k); - goFilesIn = dir: - let files = readDir dir; - goFiles = filter (f: goFileFilter f files."${f}") (attrNames files); - in map (f: dir + "/" + f) goFiles; - allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps))); xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); @@ -93,6 +87,41 @@ let # Build a Go library out of the specified gRPC definition. grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc ]; }); + # Traverse an externally defined Go library to build up a tree of + # its packages. + # + # TODO(tazjin): Automatically infer which packages depend on which + # other packages, which currently requires overriding. + # + # TODO(tazjin): Add support for rewriting package paths. + external' = { src, path, deps ? [] }: + let + dir = readDir src; + isGoFile = f: (match ".*\.go" f) != null; + isGoTest = f: (match ".*_test\.go" f) != null; + goFileFilter = k: v: (v == "regular") && (isGoFile k) && (!isGoTest k); + goSources = + let goFiles = filter (f: goFileFilter f dir."${f}") (attrNames dir); + in map (f: src + ("/" + f)) goFiles; + + subDirs = filter (n: dir."${n}" == "directory") (attrNames dir); + subPackages = map (name: { + inherit name; + value = external' { + inherit deps; + src = src + ("/" + name); + path = path + ("/" + name); + }; + }) subDirs; + subAttrs = listToAttrs (filter (p: p.value != {}) subPackages); + + current = package { + inherit deps path; + name = pathToName path; + srcs = goSources; + }; + in if goSources == [] then subAttrs else (current // subAttrs); + # Build an externally defined Go library using `go build` itself. # # Libraries built this way can be included in any standard buildGo @@ -144,4 +173,12 @@ in { proto = makeOverridable proto; grpc = makeOverridable grpc; external = makeOverridable external; + + # TODO: remove + inherit external'; + + extTest = external' { + src = /home/tazjin/go/src/cloud.google.com/go; + path = "cloud.google.com/go"; + }; } -- cgit 1.4.1 From 57f37743c2e295b0404023587b92c517e5731a30 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Wed, 11 Dec 2019 09:54:02 +0000 Subject: fix(buildGo): Ensure 'proto' libraries are overridable --- buildGo.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildGo.nix b/buildGo.nix index 366be770a959..b5a6ba2b3a53 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -73,7 +73,7 @@ let '') // { goDeps = uniqueDeps; }; # Build a Go library out of the specified protobuf definition. - proto = { name, proto, path ? name, extraDeps ? [] }: package { + proto = { name, proto, path ? name, extraDeps ? [] }: (makeOverridable package) { inherit name path; deps = [ protoLibs.goProto ] ++ extraDeps; srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} '' -- cgit 1.4.1 From e60dfabc2122bbc29fc9832d1562826c08772442 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 12 Dec 2019 03:14:27 +0000 Subject: feat(buildGo): Expose Go import path for packages --- buildGo.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildGo.nix b/buildGo.nix index b5a6ba2b3a53..c03c30bef1a0 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -70,7 +70,7 @@ let mkdir -p $out/${path} ${srcList path (map (s: "${s}") srcs)} ${go}/bin/go tool compile -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} - '') // { goDeps = uniqueDeps; }; + '') // { goDeps = uniqueDeps; goImportPath = path; }; # Build a Go library out of the specified protobuf definition. proto = { name, proto, path ? name, extraDeps ? [] }: (makeOverridable package) { -- cgit 1.4.1 From fb4dd761461de12ccbc276431ea8aa37dcefa9b0 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 12 Dec 2019 03:23:09 +0000 Subject: feat(external): Implement tool to analyse external dependencies Adds a tool that can analyse dependencies that were not originally meant to be built with buildGo.nix and return information that can be used to construct appropriate Nix dependencies. The tool will return information about package-local and foreign dependencies separately to let Nix determine whether all required dependencies are provided and to correctly link together sub-packages. To avoid listing standard library imports in the dependencies, a list of all packages in the standard library is generated statically to allow for those to be filtered out during the analysis. This tool is still work-in-progress. --- external/default.nix | 29 ++++++++++ external/main.go | 159 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 188 insertions(+) create mode 100644 external/default.nix create mode 100644 external/main.go diff --git a/external/default.nix b/external/default.nix new file mode 100644 index 000000000000..79d559c05498 --- /dev/null +++ b/external/default.nix @@ -0,0 +1,29 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 +{ runCommand, go, jq, ripgrep, program }: + +let + # Collect all non-vendored dependencies from the Go standard library + # into a file that can be used to filter them out when processing + # dependencies. + stdlibPackages = runCommand "stdlib-pkgs.json" {} '' + export GOPATH=/dev/null + ${go}/bin/go list all | \ + ${ripgrep}/bin/rg -v 'vendor' | \ + ${jq}/bin/jq -R '.' | \ + ${jq}/bin/jq -c -s 'map({key: ., value: true}) | from_entries' \ + > $out + ''; + + analyser = program { + name = "analyser"; + + srcs = [ + ./main.go + ]; + + x_defs = { + "main.stdlibList" = "${stdlibPackages}"; + }; + }; +in analyser diff --git a/external/main.go b/external/main.go new file mode 100644 index 000000000000..0f8834fc99c9 --- /dev/null +++ b/external/main.go @@ -0,0 +1,159 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 + +// This tool analyses external (i.e. not built with `buildGo.nix`) Go +// packages to determine a build plan that Nix can import. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "go/build" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" +) + +// Path to a JSON file describing all standard library import paths. +// This file is generated and set here by Nix during the build +// process. +var stdlibList string + +// pkg describes a single Go package within the specified source +// directory. +// +// Return information includes the local (relative from project root) +// and external (none-stdlib) dependencies of this package. +type pkg struct { + Name string `json:"name"` + Source string `json:"source"` + Files []string `json:"files"` + LocalDeps []string `json:"localDeps"` + ForeignDeps []string `json:"foreignDeps"` +} + +// findGoDirs returns a filepath.WalkFunc that identifies all +// directories that contain Go source code in a certain tree. +func findGoDirs(at string) ([]string, error) { + var goDirs []string + dir := "" + + err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error { + // Skip testdata + if info.IsDir() && info.Name() == "testdata" { + return filepath.SkipDir + } + + // Keep track of the last seen directory. + if info.IsDir() { + dir = path + return nil + } + + // If the directory has already been "popped", nothing else needs + // to happen. + if dir == "" { + return nil + } + + // If the current file is a Go file, then the directory is popped + // (i.e. marked as a Go directory). + if strings.HasSuffix(info.Name(), ".go") && !strings.HasSuffix(info.Name(), "_test.go") { + goDirs = append(goDirs, dir) + dir = "" + return nil + } + + return nil + }) + + if err != nil { + return nil, err + } + + return goDirs, nil +} + +// analysePackage loads and analyses the imports of a single Go +// package, returning the data that is required by the Nix code to +// generate a derivation for this package. +func analysePackage(root, source, path string, stdlib map[string]bool) (pkg, error) { + ctx := build.Default + + p, err := ctx.ImportDir(source, build.IgnoreVendor) + if err != nil { + return pkg{}, err + } + + local := []string{} + foreign := []string{} + + for _, i := range p.Imports { + if stdlib[i] { + continue + } + + if strings.HasPrefix(i, path) { + local = append(local, i) + } else { + foreign = append(foreign, i) + } + } + + analysed := pkg{ + Name: strings.TrimPrefix(source, root+"/"), + Source: source, + Files: p.GoFiles, + LocalDeps: local, + ForeignDeps: foreign, + } + + return analysed, nil +} + +func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) { + f, err := ioutil.ReadFile(from) + if err != nil { + return + } + + err = json.Unmarshal(f, &pkgs) + return +} + +func main() { + // TODO(tazjin): Remove default values + source := flag.String("source", "/nix/store/fzp67ris29zg5zfs65z0q245x0fahgll-source", "path to directory with sources to process") + path := flag.String("path", "github.com/golang/protobuf", "import path for the package") + + flag.Parse() + + if *source == "" { + log.Fatalf("-source flag must be specified") + } + + stdlibPkgs, err := loadStdlibPkgs(stdlibList) + if err != nil { + log.Fatalf("failed to load standard library index from %q: %s\n", stdlibList, err) + } + + goDirs, err := findGoDirs(*source) + if err != nil { + log.Fatalf("failed to walk source directory '%s': %s\n", source, err) + } + + all := []pkg{} + for _, d := range goDirs { + analysed, err := analysePackage(*source, d, *path, stdlibPkgs) + if err != nil { + log.Fatalf("failed to analyse package at %q: %s", d, err) + } + all = append(all, analysed) + } + + j, _ := json.MarshalIndent(all, "", " ") // TODO: no indent + fmt.Println(string(j)) +} -- cgit 1.4.1 From a5473293e78fc0b01def5c4c3f7e11854d88d8d0 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 12 Dec 2019 03:46:28 +0000 Subject: feat(external): Return references in more useable format for Nix Sub-packages of external dependencies are traversed by Nix as a tree of attribute sets which need to be accessed by "path". To make this easier, the dependency analyser now returns "paths" as string lists. --- external/main.go | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/external/main.go b/external/main.go index 0f8834fc99c9..2e5ee4900b37 100644 --- a/external/main.go +++ b/external/main.go @@ -13,6 +13,7 @@ import ( "io/ioutil" "log" "os" + "path" "path/filepath" "strings" ) @@ -28,11 +29,10 @@ var stdlibList string // Return information includes the local (relative from project root) // and external (none-stdlib) dependencies of this package. type pkg struct { - Name string `json:"name"` - Source string `json:"source"` - Files []string `json:"files"` - LocalDeps []string `json:"localDeps"` - ForeignDeps []string `json:"foreignDeps"` + Name []string `json:"name"` + Files []string `json:"files"` + LocalDeps [][]string `json:"localDeps"` + ForeignDeps []string `json:"foreignDeps"` } // findGoDirs returns a filepath.WalkFunc that identifies all @@ -80,7 +80,7 @@ func findGoDirs(at string) ([]string, error) { // analysePackage loads and analyses the imports of a single Go // package, returning the data that is required by the Nix code to // generate a derivation for this package. -func analysePackage(root, source, path string, stdlib map[string]bool) (pkg, error) { +func analysePackage(root, source, importpath string, stdlib map[string]bool) (pkg, error) { ctx := build.Default p, err := ctx.ImportDir(source, build.IgnoreVendor) @@ -88,7 +88,7 @@ func analysePackage(root, source, path string, stdlib map[string]bool) (pkg, err return pkg{}, err } - local := []string{} + local := [][]string{} foreign := []string{} for _, i := range p.Imports { @@ -96,17 +96,22 @@ func analysePackage(root, source, path string, stdlib map[string]bool) (pkg, err continue } - if strings.HasPrefix(i, path) { - local = append(local, i) + if strings.HasPrefix(i, importpath) { + local = append(local, strings.Split(strings.TrimPrefix(i, importpath+"/"), "/")) } else { foreign = append(foreign, i) } } + prefix := strings.TrimPrefix(source, root+"/") + files := []string{} + for _, f := range p.GoFiles { + files = append(files, path.Join(prefix, f)) + } + analysed := pkg{ - Name: strings.TrimPrefix(source, root+"/"), - Source: source, - Files: p.GoFiles, + Name: strings.Split(prefix, "/"), + Files: files, LocalDeps: local, ForeignDeps: foreign, } @@ -125,9 +130,8 @@ func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) { } func main() { - // TODO(tazjin): Remove default values - source := flag.String("source", "/nix/store/fzp67ris29zg5zfs65z0q245x0fahgll-source", "path to directory with sources to process") - path := flag.String("path", "github.com/golang/protobuf", "import path for the package") + source := flag.String("source", "", "path to directory with sources to process") + path := flag.String("path", "", "import path for the package") flag.Parse() @@ -154,6 +158,6 @@ func main() { all = append(all, analysed) } - j, _ := json.MarshalIndent(all, "", " ") // TODO: no indent + j, _ := json.Marshal(all) fmt.Println(string(j)) } -- cgit 1.4.1 From b20e46d60b14d5e367e52ec14989dc27b4829774 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 12 Dec 2019 16:33:10 +0000 Subject: fix(external): Ensure findGoDirs "finds" top-level directory Due to the lexical walk order of `filepath.Walk` the previous directory identification logic failed under certain conditions if the top-level directory contained Go files that showed up *after* the first subdirectories. To simplify the logic a set of directories is now gathered instead on a file-level. --- external/main.go | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/external/main.go b/external/main.go index 2e5ee4900b37..0c1d84d5b723 100644 --- a/external/main.go +++ b/external/main.go @@ -38,33 +38,18 @@ type pkg struct { // findGoDirs returns a filepath.WalkFunc that identifies all // directories that contain Go source code in a certain tree. func findGoDirs(at string) ([]string, error) { - var goDirs []string - dir := "" + dirSet := make(map[string]bool) err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error { - // Skip testdata - if info.IsDir() && info.Name() == "testdata" { + // Skip folders that are guaranteed to not be relevant + if info.IsDir() && (info.Name() == "testdata" || info.Name() == ".git") { return filepath.SkipDir } - // Keep track of the last seen directory. - if info.IsDir() { - dir = path - return nil - } - - // If the directory has already been "popped", nothing else needs - // to happen. - if dir == "" { - return nil - } - // If the current file is a Go file, then the directory is popped // (i.e. marked as a Go directory). - if strings.HasSuffix(info.Name(), ".go") && !strings.HasSuffix(info.Name(), "_test.go") { - goDirs = append(goDirs, dir) - dir = "" - return nil + if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") && !strings.HasSuffix(info.Name(), "_test.go") { + dirSet[filepath.Dir(path)] = true } return nil @@ -74,6 +59,11 @@ func findGoDirs(at string) ([]string, error) { return nil, err } + goDirs := []string{} + for k, _ := range dirSet { + goDirs = append(goDirs, k) + } + return goDirs, nil } -- cgit 1.4.1 From 1fd80fb20175c28bc500e6d889594d1a5ebb6be7 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 12 Dec 2019 17:04:40 +0000 Subject: fix(external): Correctly set names for root packages Fixes the prefix trimming logic for package names and source files if the source files appear in the package root (which is, unsurprisingly, very common). --- external/main.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/external/main.go b/external/main.go index 0c1d84d5b723..028703e38cad 100644 --- a/external/main.go +++ b/external/main.go @@ -94,19 +94,27 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk } prefix := strings.TrimPrefix(source, root+"/") + + name := []string{} + if len(prefix) != len(source) { + name = strings.Split(prefix, "/") + } else { + // Otherwise, the name is empty since its the root package and no + // prefix should be added to files. + prefix = "" + } + files := []string{} for _, f := range p.GoFiles { files = append(files, path.Join(prefix, f)) } - analysed := pkg{ - Name: strings.Split(prefix, "/"), + return pkg{ + Name: name, Files: files, LocalDeps: local, ForeignDeps: foreign, - } - - return analysed, nil + }, nil } func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) { -- cgit 1.4.1 From bbf3a418a5a119e2ab6aaaf1e04f42caa1b683a8 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 12 Dec 2019 22:58:51 +0000 Subject: feat(external): Add fully qualified import path to analyser output This is used by Nix to build the derivation names for individual packages. --- external/main.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/external/main.go b/external/main.go index 028703e38cad..aac966172b34 100644 --- a/external/main.go +++ b/external/main.go @@ -29,7 +29,8 @@ var stdlibList string // Return information includes the local (relative from project root) // and external (none-stdlib) dependencies of this package. type pkg struct { - Name []string `json:"name"` + Name string `json:"name"` + Locator []string `json:"locator"` Files []string `json:"files"` LocalDeps [][]string `json:"localDeps"` ForeignDeps []string `json:"foreignDeps"` @@ -95,12 +96,12 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk prefix := strings.TrimPrefix(source, root+"/") - name := []string{} + locator := []string{} if len(prefix) != len(source) { - name = strings.Split(prefix, "/") + locator = strings.Split(prefix, "/") } else { - // Otherwise, the name is empty since its the root package and no - // prefix should be added to files. + // Otherwise, the locator is empty since its the root package and + // no prefix should be added to files. prefix = "" } @@ -110,7 +111,8 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk } return pkg{ - Name: name, + Name: path.Join(importpath, prefix), + Locator: locator, Files: files, LocalDeps: local, ForeignDeps: foreign, -- cgit 1.4.1 From c5373a69fe440bada29f7e2fc41882890f7d0872 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 12 Dec 2019 23:00:20 +0000 Subject: feat(external): Implement builder function for externals Implements a builder function that calls the analysis tool on the provided source and builds up the required attribute set, including local dependencies. --- external/default.nix | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/external/default.nix b/external/default.nix index 79d559c05498..a065ab3331aa 100644 --- a/external/default.nix +++ b/external/default.nix @@ -1,8 +1,13 @@ # Copyright 2019 Google LLC. # SPDX-License-Identifier: Apache-2.0 -{ runCommand, go, jq, ripgrep, program }: +{ pkgs, program, package }: let + inherit (builtins) foldl'fromJSON head readFile replaceStrings tail throw; + inherit (pkgs) lib runCommand go jq ripgrep; + + pathToName = p: replaceStrings ["/"] ["_"] (toString p); + # Collect all non-vendored dependencies from the Go standard library # into a file that can be used to filter them out when processing # dependencies. @@ -26,4 +31,26 @@ let "main.stdlibList" = "${stdlibPackages}"; }; }; -in analyser + + mkset = path: value: + if path == [] then { gopkg = value; } + else { "${head path}" = mkset (tail path) value; }; + + toPackage = self: src: path: entry: package { + name = pathToName entry.name entry.name; + path = lib.concatStringsSep "/" ([ path ] ++ entry.locator); + srcs = map (f: src + ("/" + f)) entry.files; + deps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) ( + throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'" + ) self) entry.localDeps; + }; + +in { src, path, deps ? [] }: let + name = pathToName path; + analysisOutput = runCommand "${name}-structure.json" {} '' + ${analyser}/bin/analyser -path ${path} -source ${src} > $out + ''; + analysis = fromJSON (readFile analysisOutput); +in lib.fix(self: foldl' lib.recursiveUpdate {} ( + map (entry: mkset entry.locator (toPackage self src path entry)) analysis +)) -- cgit 1.4.1 From f5e3183de14e73ab6ff0ada0a6d2e9408a5280be Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 12 Dec 2019 23:07:42 +0000 Subject: feat(buildGo): Expose new external builder --- buildGo.nix | 101 +++++++++--------------------------------------------------- 1 file changed, 14 insertions(+), 87 deletions(-) diff --git a/buildGo.nix b/buildGo.nix index c03c30bef1a0..a470c1db5f03 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -72,10 +72,23 @@ let ${go}/bin/go tool compile -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} '') // { goDeps = uniqueDeps; goImportPath = path; }; + # Build a tree of Go libraries out of an external Go source + # directory that follows the standard Go layout and was not built + # with buildGo.nix. + # + # The derivation for each actual dependency will reside in an + # attribute named "gopkg". + external = import ./external { inherit pkgs program package; }; + + # Import support libraries needed for protobuf & gRPC support + protoLibs = import ./proto.nix { + inherit external; + }; + # Build a Go library out of the specified protobuf definition. proto = { name, proto, path ? name, extraDeps ? [] }: (makeOverridable package) { inherit name path; - deps = [ protoLibs.goProto ] ++ extraDeps; + deps = [ protoLibs'.protobuf ] ++ extraDeps; srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} '' cp ${proto} ${baseNameOf proto} ${protobuf}/bin/protoc --plugin=${protoLibs.goProto}/bin/protoc-gen-go \ @@ -87,84 +100,6 @@ let # Build a Go library out of the specified gRPC definition. grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc ]; }); - # Traverse an externally defined Go library to build up a tree of - # its packages. - # - # TODO(tazjin): Automatically infer which packages depend on which - # other packages, which currently requires overriding. - # - # TODO(tazjin): Add support for rewriting package paths. - external' = { src, path, deps ? [] }: - let - dir = readDir src; - isGoFile = f: (match ".*\.go" f) != null; - isGoTest = f: (match ".*_test\.go" f) != null; - goFileFilter = k: v: (v == "regular") && (isGoFile k) && (!isGoTest k); - goSources = - let goFiles = filter (f: goFileFilter f dir."${f}") (attrNames dir); - in map (f: src + ("/" + f)) goFiles; - - subDirs = filter (n: dir."${n}" == "directory") (attrNames dir); - subPackages = map (name: { - inherit name; - value = external' { - inherit deps; - src = src + ("/" + name); - path = path + ("/" + name); - }; - }) subDirs; - subAttrs = listToAttrs (filter (p: p.value != {}) subPackages); - - current = package { - inherit deps path; - name = pathToName path; - srcs = goSources; - }; - in if goSources == [] then subAttrs else (current // subAttrs); - - # Build an externally defined Go library using `go build` itself. - # - # Libraries built this way can be included in any standard buildGo - # build. - # - # Contrary to other functions, `src` is expected to point at a - # single directory containing the root of the external library. - external = { path, src, deps ? [], srcOnly ? false, targets ? [ "..." ] }: - let - name = pathToName path; - uniqueDeps = allDeps deps; - srcDir = runCommand "goext-src-${name}" {} '' - mkdir -p $out/${dirOf path} - cp -r ${src} $out/${dirOf path}/${baseNameOf path} - ''; - gopathSrc = symlinkJoin { - name = "gopath-${name}"; - paths = uniqueDeps ++ [ srcDir ]; - }; - gopathPkg = runCommand "goext-pkg-${name}" {} '' - mkdir -p gopath $out - export GOPATH=$PWD/gopath - ln -s ${gopathSrc} gopath/src - ${go}/bin/go install ${spaceOut (map (t: path + "/" + t) targets)} - - if [[ -d gopath/pkg/linux_amd64 ]]; then - echo "Installing Go packages for ${path}" - mv gopath/pkg/linux_amd64/* $out - fi - - if [[ -d gopath/bin ]]; then - echo "Installing Go binaries for ${path}" - mv gopath/bin $out/bin - fi - ''; - in (if srcOnly then gopathSrc else symlinkJoin { - name = "goext-${name}"; - paths = [ gopathSrc gopathPkg ]; - }) // { goDeps = uniqueDeps; }; - - protoLibs = import ./proto.nix { - inherit external; - }; in { # Only the high-level builder functions are exposed, but made # overrideable. @@ -173,12 +108,4 @@ in { proto = makeOverridable proto; grpc = makeOverridable grpc; external = makeOverridable external; - - # TODO: remove - inherit external'; - - extTest = external' { - src = /home/tazjin/go/src/cloud.google.com/go; - path = "cloud.google.com/go"; - }; } -- cgit 1.4.1 From 875628a0978937d708ce9365c9cc2061f39b0bca Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 12 Dec 2019 23:26:08 +0000 Subject: feat(external): Switch between packages & programs automatically --- external/default.nix | 28 +++++++++++++++++++--------- external/main.go | 2 ++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/external/default.nix b/external/default.nix index a065ab3331aa..5854d4f0277f 100644 --- a/external/default.nix +++ b/external/default.nix @@ -3,7 +3,7 @@ { pkgs, program, package }: let - inherit (builtins) foldl'fromJSON head readFile replaceStrings tail throw; + inherit (builtins) elemAt foldl' fromJSON head length readFile replaceStrings tail throw; inherit (pkgs) lib runCommand go jq ripgrep; pathToName = p: replaceStrings ["/"] ["_"] (toString p); @@ -36,14 +36,24 @@ let if path == [] then { gopkg = value; } else { "${head path}" = mkset (tail path) value; }; - toPackage = self: src: path: entry: package { - name = pathToName entry.name entry.name; - path = lib.concatStringsSep "/" ([ path ] ++ entry.locator); - srcs = map (f: src + ("/" + f)) entry.files; - deps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) ( - throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'" - ) self) entry.localDeps; - }; + last = l: elemAt l ((length l) - 1); + + toPackage = self: src: path: entry: + let + args = { + srcs = map (f: src + ("/" + f)) entry.files; + deps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) ( + throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'" + ) self) entry.localDeps; + }; + libArgs = args // { + name = pathToName entry.name; + path = lib.concatStringsSep "/" ([ path ] ++ entry.locator); + }; + binArgs = args // { + name = last ([ path ] ++ entry.locator); + }; + in if entry.isCommand then (program binArgs) else (package libArgs); in { src, path, deps ? [] }: let name = pathToName path; diff --git a/external/main.go b/external/main.go index aac966172b34..23fd53326ed0 100644 --- a/external/main.go +++ b/external/main.go @@ -34,6 +34,7 @@ type pkg struct { Files []string `json:"files"` LocalDeps [][]string `json:"localDeps"` ForeignDeps []string `json:"foreignDeps"` + IsCommand bool `json:"isCommand"` } // findGoDirs returns a filepath.WalkFunc that identifies all @@ -116,6 +117,7 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk Files: files, LocalDeps: local, ForeignDeps: foreign, + IsCommand: p.IsCommand(), }, nil } -- cgit 1.4.1 From 0dfa5d15fae5b8f5992625f03586ba6043bcd475 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 12 Dec 2019 23:27:43 +0000 Subject: fix(buildGo): Update buildGo.proto for compatibility with external Changes in the structure of buildGo.external meant that the package layout for the protobuf library is now slightly different. `proto` has been amended to work with the new structure. Callers of buildGo.proto do not need to be updated (i.e. the example still works). --- buildGo.nix | 8 ++++---- proto.nix | 4 ---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/buildGo.nix b/buildGo.nix index a470c1db5f03..0df3c955aef1 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -76,8 +76,8 @@ let # directory that follows the standard Go layout and was not built # with buildGo.nix. # - # The derivation for each actual dependency will reside in an - # attribute named "gopkg". + # The derivation for each actual package will reside in an attribute + # named "gopkg", and an attribute named "gobin" for binaries. external = import ./external { inherit pkgs program package; }; # Import support libraries needed for protobuf & gRPC support @@ -88,10 +88,10 @@ let # Build a Go library out of the specified protobuf definition. proto = { name, proto, path ? name, extraDeps ? [] }: (makeOverridable package) { inherit name path; - deps = [ protoLibs'.protobuf ] ++ extraDeps; + deps = [ protoLibs.goProto.proto.gopkg ] ++ extraDeps; srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} '' cp ${proto} ${baseNameOf proto} - ${protobuf}/bin/protoc --plugin=${protoLibs.goProto}/bin/protoc-gen-go \ + ${protobuf}/bin/protoc --plugin=${protoLibs.goProto.protoc-gen-go.gopkg}/bin/protoc-gen-go \ --go_out=plugins=grpc,import_path=${baseNameOf path}:. ${baseNameOf proto} mv *.pb.go $out ''); diff --git a/proto.nix b/proto.nix index e773af031873..dd40def0ff1f 100644 --- a/proto.nix +++ b/proto.nix @@ -19,7 +19,6 @@ in rec { xnet = external { path = "golang.org/x/net"; - srcOnly = true; deps = [ xtext ]; src = fetchGit { url = "https://go.googlesource.com/net"; @@ -29,7 +28,6 @@ in rec { xsys = external { path = "golang.org/x/sys"; - srcOnly = true; src = fetchGit { url = "https://go.googlesource.com/sys"; rev = "bd437916bb0eb726b873ee8e9b2dcf212d32e2fd"; @@ -38,7 +36,6 @@ in rec { xtext = external { path = "golang.org/x/text"; - srcOnly = true; src = fetchGit { url = "https://go.googlesource.com/text"; rev = "cbf43d21aaebfdfeb81d91a5f444d13a3046e686"; @@ -47,7 +44,6 @@ in rec { genproto = external { path = "google.golang.org/genproto"; - srcOnly = true; src = fetchGit { url = "https://github.com/google/go-genproto"; rev = "83cc0476cb11ea0da33dacd4c6354ab192de6fe6"; -- cgit 1.4.1 From 39e73f42b21e369906442fc3235cf9ce62f25ce1 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 12 Dec 2019 23:30:47 +0000 Subject: style(external): Minor formatting fixes --- external/default.nix | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/external/default.nix b/external/default.nix index 5854d4f0277f..f9cb542e6f14 100644 --- a/external/default.nix +++ b/external/default.nix @@ -3,7 +3,17 @@ { pkgs, program, package }: let - inherit (builtins) elemAt foldl' fromJSON head length readFile replaceStrings tail throw; + inherit (builtins) + elemAt + foldl' + fromJSON + head + length + readFile + replaceStrings + tail + throw; + inherit (pkgs) lib runCommand go jq ripgrep; pathToName = p: replaceStrings ["/"] ["_"] (toString p); -- cgit 1.4.1 From 218b81c709248784112ae7bd0646c7fc6773fe5f Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 12 Dec 2019 23:53:39 +0000 Subject: fix(external): Skip folders with "no buildable Go files" This error is returned by the build analysis logic if the target package does not have any relevant Go files, which might be the case if `+build` flags and such are used. --- external/main.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/external/main.go b/external/main.go index 23fd53326ed0..e0b95605f0dd 100644 --- a/external/main.go +++ b/external/main.go @@ -43,14 +43,15 @@ func findGoDirs(at string) ([]string, error) { dirSet := make(map[string]bool) err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error { + name := info.Name() // Skip folders that are guaranteed to not be relevant - if info.IsDir() && (info.Name() == "testdata" || info.Name() == ".git") { + if info.IsDir() && (name == "testdata" || name == ".git") { return filepath.SkipDir } // If the current file is a Go file, then the directory is popped // (i.e. marked as a Go directory). - if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") && !strings.HasSuffix(info.Name(), "_test.go") { + if !info.IsDir() && strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go") { dirSet[filepath.Dir(path)] = true } @@ -154,6 +155,16 @@ func main() { all := []pkg{} for _, d := range goDirs { analysed, err := analysePackage(*source, d, *path, stdlibPkgs) + + // If the Go source analysis returned "no buildable Go files", + // that directory should be skipped. + // + // This might be due to `+build` flags on the platform and other + // reasons (such as test files). + if _, ok := err.(*build.NoGoError); ok { + continue + } + if err != nil { log.Fatalf("failed to analyse package at %q: %s", d, err) } -- cgit 1.4.1 From d8ef203f78ba58968d26fb3a749cb32170394500 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 13 Dec 2019 00:07:47 +0000 Subject: feat(external): Support foreign dependencies in external packages Users can supply a list of foreign dependencies in calls to buildGo.external. These are now appropriately inserted into packages that *need them* and no further, resolving issues with complex internal recursion in some repositories! --- external/default.nix | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/external/default.nix b/external/default.nix index f9cb542e6f14..49f8bd93149a 100644 --- a/external/default.nix +++ b/external/default.nix @@ -9,6 +9,7 @@ let fromJSON head length + listToAttrs readFile replaceStrings tail @@ -48,29 +49,45 @@ let last = l: elemAt l ((length l) - 1); - toPackage = self: src: path: entry: + toPackage = self: src: path: depMap: entry: let + localDeps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) ( + throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'" + ) self) entry.localDeps; + + foreignDeps = map (d: lib.attrByPath [ d ] ( + throw "missing foreign dependency '${d}' in '${path}'" + ) depMap) entry.foreignDeps; + args = { srcs = map (f: src + ("/" + f)) entry.files; - deps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) ( - throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'" - ) self) entry.localDeps; + deps = localDeps ++ foreignDeps; }; + libArgs = args // { name = pathToName entry.name; path = lib.concatStringsSep "/" ([ path ] ++ entry.locator); }; + binArgs = args // { name = last ([ path ] ++ entry.locator); }; in if entry.isCommand then (program binArgs) else (package libArgs); in { src, path, deps ? [] }: let + # Build a map of dependencies (from their import paths to their + # derivation) so that they can be conditionally imported only in + # sub-packages that require them. + depMap = listToAttrs (map (d: { + name = d.goImportPath; + value = d; + }) deps); + name = pathToName path; analysisOutput = runCommand "${name}-structure.json" {} '' ${analyser}/bin/analyser -path ${path} -source ${src} > $out ''; analysis = fromJSON (readFile analysisOutput); in lib.fix(self: foldl' lib.recursiveUpdate {} ( - map (entry: mkset entry.locator (toPackage self src path entry)) analysis + map (entry: mkset entry.locator (toPackage self src path depMap entry)) analysis )) -- cgit 1.4.1 From 41c28cf60ec27afde7379d2389ce8b5b7d420c5e Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 13 Dec 2019 00:08:36 +0000 Subject: fix(buildGo): Correctly refer to goGrpc package --- buildGo.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildGo.nix b/buildGo.nix index 0df3c955aef1..759a671fad9b 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -98,7 +98,7 @@ let }; # Build a Go library out of the specified gRPC definition. - grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc ]; }); + grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc.gopkg ]; }); in { # Only the high-level builder functions are exposed, but made -- cgit 1.4.1 From 369c86e0ef8a713d47c2a6b94a3f70d527b84a3c Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 13 Dec 2019 00:08:56 +0000 Subject: fix(proto): Refactor gRPC dependencies to match new external layout This is now a lot more fine-grained than before, but it actually works fine. This stuff is a bit annoying to write by hand. There are multiple different options available (e.g. carrying an attribute set of all subpackages in each `external` and only passing that, having a tool generate this, etc.). --- proto.nix | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/proto.nix b/proto.nix index dd40def0ff1f..2ece948ebd84 100644 --- a/proto.nix +++ b/proto.nix @@ -19,11 +19,17 @@ in rec { xnet = external { path = "golang.org/x/net"; - deps = [ xtext ]; + src = fetchGit { url = "https://go.googlesource.com/net"; rev = "ffdde105785063a81acd95bdf89ea53f6e0aac2d"; }; + + deps = map (p: p.gopkg) [ + xtext.secure.bidirule + xtext.unicode.bidi + xtext.unicode.norm + ]; }; xsys = external { @@ -48,21 +54,31 @@ in rec { url = "https://github.com/google/go-genproto"; rev = "83cc0476cb11ea0da33dacd4c6354ab192de6fe6"; }; + + deps = with goProto; map (p: p.gopkg) [ + proto + ptypes.any + ]; }; goGrpc = external { path = "google.golang.org/grpc"; - deps = [ goProto xnet xsys genproto ]; + deps = map (p: p.gopkg) ([ + xnet.trace + xnet.http2 + xsys.unix + xnet.http2.hpack + genproto.googleapis.rpc.status + ] ++ (with goProto; [ + proto + ptypes + ptypes.duration + ptypes.timestamp + ])); src = fetchGit { url = "https://github.com/grpc/grpc-go"; rev = "d8e3da36ac481ef00e510ca119f6b68177713689"; }; - - targets = [ - "." - "codes" - "status" - ]; }; } -- cgit 1.4.1 From 9d5417501b1bdf5f363221a14b2e2a92ef2ba3e2 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 13 Dec 2019 00:12:33 +0000 Subject: docs(README): Update description of buildGo.external --- README.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 489cf128173f..e84ede663bf8 100644 --- a/README.md +++ b/README.md @@ -90,19 +90,20 @@ in buildGo.program { | `deps` | `list` | List of dependencies (i.e. other Go libraries) | no | | `path` | `string` | Go import path for the resulting library | no | -* `buildGo.external`: Build a Go library or program using standard `go` tooling. - - This exists for compatibility with complex external dependencies. In theory it - is possible to write `buildGo.package` specifications for each subpackage of - an external dependency, but it is often cumbersome to do so. - - | parameter | type | use | required? | - |-----------|----------------|------------------------------------------------|-----------| - | `path` | `string` | Go import path for the resulting library | yes | - | `src` | `path` | Path to the source **directory** | yes | - | `deps` | `list` | List of dependencies (i.e. other Go libraries) | no | - | `srcOnly` | `bool` | Only copy sources, do not perform a build. | no | - | `targets` | `list` | Sub-packages to build (defaults to all) | no | +* `buildGo.external`: Build an externally defined Go library or program. + + This function performs analysis on the supplied source code (which + can use the standard Go tooling layout) and creates a tree of all + the packages contained within. + + This exists for compatibility with external libraries that were not + defined using buildGo. + + | parameter | type | use | required? | + |-----------|----------------|-----------------------------------------------|-----------| + | `path` | `string` | Go import path for the resulting package | yes | + | `src` | `path` | Path to the source **directory** | yes | + | `deps` | `list` | List of dependencies (i.e. other Go packages) | no | For some examples of how `buildGo.external` is used, check out [`proto.nix`](./proto.nix). -- cgit 1.4.1 From 5649c75d01dc9058d9a6092ad2a504669cc780a9 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 13 Dec 2019 12:10:32 +0000 Subject: fix(external): Fix "inverted" local dependencies Usually in large packages the root depends on one or more sub-packages (or there is no root), but some projects (e.g. golang.org/x/oauth2) do it the other way around. This fix adds compatibility for both ways. --- external/main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/external/main.go b/external/main.go index e0b95605f0dd..d73f1540a986 100644 --- a/external/main.go +++ b/external/main.go @@ -89,7 +89,9 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk continue } - if strings.HasPrefix(i, importpath) { + if i == importpath { + local = append(local, []string{}) + } else if strings.HasPrefix(i, importpath) { local = append(local, strings.Split(strings.TrimPrefix(i, importpath+"/"), "/")) } else { foreign = append(foreign, i) -- cgit 1.4.1 From cfae527cc14c55f2cfa3d356674971c627ad1a88 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 13 Dec 2019 15:15:22 +0000 Subject: feat(buildGo): Support linking of symabi files for ASM sources --- buildGo.nix | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/buildGo.nix b/buildGo.nix index 759a671fad9b..b61c3904fb18 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -64,12 +64,17 @@ let # # This outputs both the sources and compiled binary, as both are # needed when downstream packages depend on it. - package = { name, srcs, deps ? [], path ? name }: + package = { name, srcs, deps ? [], path ? name, sfiles ? [] }: let uniqueDeps = allDeps deps; + asmBuild = if sfiles == [] then "" else '' + ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles} + ''; + asmLink = if sfiles == [] then "-complete" else "-symabis ./symabis"; in (runCommand "golib-${name}" {} '' mkdir -p $out/${path} ${srcList path (map (s: "${s}") srcs)} - ${go}/bin/go tool compile -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} + ${asmBuild} + ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} '') // { goDeps = uniqueDeps; goImportPath = path; }; # Build a tree of Go libraries out of an external Go source -- cgit 1.4.1 From 859c429b2fbf21e3ea799b1068daa6f123c59a48 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 13 Dec 2019 15:15:53 +0000 Subject: feat(external): Include *.s (ASM) files in external builds --- external/default.nix | 1 + external/main.go | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/external/default.nix b/external/default.nix index 49f8bd93149a..ee0bf94fa2c7 100644 --- a/external/default.nix +++ b/external/default.nix @@ -67,6 +67,7 @@ let libArgs = args // { name = pathToName entry.name; path = lib.concatStringsSep "/" ([ path ] ++ entry.locator); + sfiles = map (f: src + ("/" + f)) entry.sfiles; }; binArgs = args // { diff --git a/external/main.go b/external/main.go index d73f1540a986..aa4a813d32bd 100644 --- a/external/main.go +++ b/external/main.go @@ -32,6 +32,7 @@ type pkg struct { Name string `json:"name"` Locator []string `json:"locator"` Files []string `json:"files"` + SFiles []string `json:"sfiles"` LocalDeps [][]string `json:"localDeps"` ForeignDeps []string `json:"foreignDeps"` IsCommand bool `json:"isCommand"` @@ -75,6 +76,7 @@ func findGoDirs(at string) ([]string, error) { // generate a derivation for this package. func analysePackage(root, source, importpath string, stdlib map[string]bool) (pkg, error) { ctx := build.Default + ctx.CgoEnabled = false p, err := ctx.ImportDir(source, build.IgnoreVendor) if err != nil { @@ -114,10 +116,16 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk files = append(files, path.Join(prefix, f)) } + sfiles := []string{} + for _, f := range p.SFiles { + sfiles = append(sfiles, path.Join(prefix, f)) + } + return pkg{ Name: path.Join(importpath, prefix), Locator: locator, Files: files, + SFiles: sfiles, LocalDeps: local, ForeignDeps: foreign, IsCommand: p.IsCommand(), -- cgit 1.4.1 From 7f74980457df843aea542510a406f34366e8b868 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 13 Dec 2019 21:23:41 +0000 Subject: feat(external): Compile Go assembly and include it in pkg archive This was the final step required to add support for packages that make use of Go assembly, such as golang.org/x/sys. --- buildGo.nix | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/buildGo.nix b/buildGo.nix index b61c3904fb18..140cbf2d9d16 100644 --- a/buildGo.nix +++ b/buildGo.nix @@ -65,16 +65,28 @@ let # This outputs both the sources and compiled binary, as both are # needed when downstream packages depend on it. package = { name, srcs, deps ? [], path ? name, sfiles ? [] }: - let uniqueDeps = allDeps deps; - asmBuild = if sfiles == [] then "" else '' - ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles} - ''; - asmLink = if sfiles == [] then "-complete" else "-symabis ./symabis"; + let + uniqueDeps = allDeps deps; + + # The build steps below need to be executed conditionally for Go + # assembly if the analyser detected any *.s files. + # + # This is required for several popular packages (e.g. x/sys). + ifAsm = do: if sfiles == [] then "" else do; + asmBuild = ifAsm '' + ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles} + ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles} + ''; + asmLink = ifAsm "-symabis ./symabis -asmhdr $out/go_asm.h"; + asmPack = ifAsm '' + ${go}/bin/go tool pack r $out/${path}.a ./asm.o + ''; in (runCommand "golib-${name}" {} '' mkdir -p $out/${path} ${srcList path (map (s: "${s}") srcs)} ${asmBuild} ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} + ${asmPack} '') // { goDeps = uniqueDeps; goImportPath = path; }; # Build a tree of Go libraries out of an external Go source -- cgit 1.4.1 From 53630e3b1544de38f509e48ac0f589bcfafa7728 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 15 Dec 2019 15:57:10 +0000 Subject: fix(external): Correctly determine binary name if locator is empty If the root of a project is a binary, the previous logic would generate invalid names. This ensure that the last path component of the name is used. --- external/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/default.nix b/external/default.nix index ee0bf94fa2c7..6e1dce11140b 100644 --- a/external/default.nix +++ b/external/default.nix @@ -71,7 +71,7 @@ let }; binArgs = args // { - name = last ([ path ] ++ entry.locator); + name = (last ((lib.splitString "/" path) ++ entry.locator)); }; in if entry.isCommand then (program binArgs) else (package libArgs); -- cgit 1.4.1 From a8fbdc52c2d7170bc8b144bef9d522bef9f0d815 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 19 Dec 2019 15:22:35 +0000 Subject: chore(buildGo): Relayout for depot merge --- README.md | 140 ------------------------- buildGo.nix | 128 ----------------------- example/default.nix | 47 --------- example/lib.go | 9 -- example/main.go | 25 ----- example/thing.proto | 10 -- external/default.nix | 94 ----------------- external/main.go | 186 --------------------------------- overrides/buildGo/README.md | 140 +++++++++++++++++++++++++ overrides/buildGo/default.nix | 128 +++++++++++++++++++++++ overrides/buildGo/example/default.nix | 47 +++++++++ overrides/buildGo/example/lib.go | 9 ++ overrides/buildGo/example/main.go | 25 +++++ overrides/buildGo/example/thing.proto | 10 ++ overrides/buildGo/external/default.nix | 94 +++++++++++++++++ overrides/buildGo/external/main.go | 186 +++++++++++++++++++++++++++++++++ overrides/buildGo/proto.nix | 84 +++++++++++++++ proto.nix | 84 --------------- 18 files changed, 723 insertions(+), 723 deletions(-) delete mode 100644 README.md delete mode 100644 buildGo.nix delete mode 100644 example/default.nix delete mode 100644 example/lib.go delete mode 100644 example/main.go delete mode 100644 example/thing.proto delete mode 100644 external/default.nix delete mode 100644 external/main.go create mode 100644 overrides/buildGo/README.md create mode 100644 overrides/buildGo/default.nix create mode 100644 overrides/buildGo/example/default.nix create mode 100644 overrides/buildGo/example/lib.go create mode 100644 overrides/buildGo/example/main.go create mode 100644 overrides/buildGo/example/thing.proto create mode 100644 overrides/buildGo/external/default.nix create mode 100644 overrides/buildGo/external/main.go create mode 100644 overrides/buildGo/proto.nix delete mode 100644 proto.nix diff --git a/README.md b/README.md deleted file mode 100644 index e84ede663bf8..000000000000 --- a/README.md +++ /dev/null @@ -1,140 +0,0 @@ -buildGo.nix -=========== - -This is an alternative [Nix][] build system for [Go][]. It supports building Go -libraries and programs, and even automatically generating Protobuf & gRPC -libraries. - -*Note:* This will probably end up being folded into [Nixery][]. - -## Background - -Most language-specific Nix tooling outsources the build to existing -language-specific build tooling, which essentially means that Nix ends up being -a wrapper around all sorts of external build systems. - -However, systems like [Bazel][] take an alternative approach in which the -compiler is invoked directly and the composition of programs and libraries stays -within a single homogeneous build system. - -Users don't need to learn per-language build systems and especially for -companies with large monorepo-setups ([like Google][]) this has huge -productivity impact. - -This project is an attempt to prove that Nix can be used in a similar style to -build software directly, rather than shelling out to other build systems. - -## Example - -Given a program layout like this: - -``` -. -├── lib <-- some library component -│   ├── bar.go -│   └── foo.go -├── api.proto <-- gRPC API definition -├── main.go <-- program implementation -└── default.nix <-- build instructions -``` - -The contents of `default.nix` could look like this: - -```nix -{ buildGo }: - -let - api = buildGo.grpc { - name = "someapi"; - proto = ./api.proto; - }; - - lib = buildGo.package { - name = "somelib"; - srcs = [ - ./lib/bar.go - ./lib/foo.go - ]; - }; -in buildGo.program { - name = "my-program"; - deps = [ api lib ]; - - srcs = [ - ./main.go - ]; -} -``` - -(If you don't know how to read Nix, check out [nix-1p][]) - -## Usage - -`buildGo` exposes five different functions: - -* `buildGo.program`: Build a Go binary out of the specified source files. - - | parameter | type | use | required? | - |-----------|-------------------------|------------------------------------------------|-----------| - | `name` | `string` | Name of the program (and resulting executable) | yes | - | `srcs` | `list` | List of paths to source files | yes | - | `deps` | `list` | List of dependencies (i.e. other Go libraries) | no | - | `x_defs` | `attrs` | Attribute set of linker vars (i.e. `-X`-flags) | no | - -* `buildGo.package`: Build a Go library out of the specified source files. - - | parameter | type | use | required? | - |-----------|--------------|------------------------------------------------|-----------| - | `name` | `string` | Name of the library (and resulting executable) | yes | - | `srcs` | `list` | List of paths to source files | yes | - | `deps` | `list` | List of dependencies (i.e. other Go libraries) | no | - | `path` | `string` | Go import path for the resulting library | no | - -* `buildGo.external`: Build an externally defined Go library or program. - - This function performs analysis on the supplied source code (which - can use the standard Go tooling layout) and creates a tree of all - the packages contained within. - - This exists for compatibility with external libraries that were not - defined using buildGo. - - | parameter | type | use | required? | - |-----------|----------------|-----------------------------------------------|-----------| - | `path` | `string` | Go import path for the resulting package | yes | - | `src` | `path` | Path to the source **directory** | yes | - | `deps` | `list` | List of dependencies (i.e. other Go packages) | no | - - For some examples of how `buildGo.external` is used, check out - [`proto.nix`](./proto.nix). - -* `buildGo.proto`: Build a Go library out of the specified Protobuf definition. - - | parameter | type | use | required? | - |-------------|-------------|--------------------------------------------------|-----------| - | `name` | `string` | Name for the resulting library | yes | - | `proto` | `path` | Path to the Protobuf definition file | yes | - | `path` | `string` | Import path for the resulting Go library | no | - | `extraDeps` | `list` | Additional Go dependencies to add to the library | no | - -* `buildGo.grpc`: Build a Go library out of the specified gRPC definition. - - The parameters are identical to `buildGo.proto`. - -## Current status - -This project is work-in-progress. Crucially it is lacking the following features: - -* feature flag parity with Bazel's Go rules -* documentation building -* test execution - -There are still some open questions around how to structure some of those -features in Nix. - -[Nix]: https://nixos.org/nix/ -[Go]: https://golang.org/ -[Nixery]: https://github.com/google/nixery -[Bazel]: https://bazel.build/ -[like Google]: https://ai.google/research/pubs/pub45424 -[nix-1p]: https://github.com/tazjin/nix-1p diff --git a/buildGo.nix b/buildGo.nix deleted file mode 100644 index 140cbf2d9d16..000000000000 --- a/buildGo.nix +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2019 Google LLC. -# SPDX-License-Identifier: Apache-2.0 -# -# buildGo provides Nix functions to build Go packages in the style of Bazel's -# rules_go. - -{ pkgs ? import {} -, ... }: - -let - inherit (builtins) - attrNames - baseNameOf - dirOf - elemAt - filter - listToAttrs - map - match - readDir - replaceStrings - toString; - - inherit (pkgs) lib go runCommand fetchFromGitHub protobuf symlinkJoin; - - # Helpers for low-level Go compiler invocations - spaceOut = lib.concatStringsSep " "; - - includeDepSrc = dep: "-I ${dep}"; - includeSources = deps: spaceOut (map includeDepSrc deps); - - includeDepLib = dep: "-L ${dep}"; - includeLibs = deps: spaceOut (map includeDepLib deps); - - srcBasename = src: elemAt (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src)) 1; - srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}"; - srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs); - - allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps))); - - xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); - - pathToName = p: replaceStrings ["/"] ["_"] (toString p); - - # Add an `overrideGo` attribute to a function result that works - # similar to `overrideAttrs`, but is used specifically for the - # arguments passed to Go builders. - makeOverridable = f: orig: (f orig) // { - overrideGo = new: makeOverridable f (orig // (new orig)); - }; - - # High-level build functions - - # Build a Go program out of the specified files and dependencies. - program = { name, srcs, deps ? [], x_defs ? {} }: - let uniqueDeps = allDeps deps; - in runCommand name {} '' - ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs} - mkdir -p $out/bin - ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a - ''; - - # Build a Go library assembled out of the specified files. - # - # This outputs both the sources and compiled binary, as both are - # needed when downstream packages depend on it. - package = { name, srcs, deps ? [], path ? name, sfiles ? [] }: - let - uniqueDeps = allDeps deps; - - # The build steps below need to be executed conditionally for Go - # assembly if the analyser detected any *.s files. - # - # This is required for several popular packages (e.g. x/sys). - ifAsm = do: if sfiles == [] then "" else do; - asmBuild = ifAsm '' - ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles} - ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles} - ''; - asmLink = ifAsm "-symabis ./symabis -asmhdr $out/go_asm.h"; - asmPack = ifAsm '' - ${go}/bin/go tool pack r $out/${path}.a ./asm.o - ''; - in (runCommand "golib-${name}" {} '' - mkdir -p $out/${path} - ${srcList path (map (s: "${s}") srcs)} - ${asmBuild} - ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} - ${asmPack} - '') // { goDeps = uniqueDeps; goImportPath = path; }; - - # Build a tree of Go libraries out of an external Go source - # directory that follows the standard Go layout and was not built - # with buildGo.nix. - # - # The derivation for each actual package will reside in an attribute - # named "gopkg", and an attribute named "gobin" for binaries. - external = import ./external { inherit pkgs program package; }; - - # Import support libraries needed for protobuf & gRPC support - protoLibs = import ./proto.nix { - inherit external; - }; - - # Build a Go library out of the specified protobuf definition. - proto = { name, proto, path ? name, extraDeps ? [] }: (makeOverridable package) { - inherit name path; - deps = [ protoLibs.goProto.proto.gopkg ] ++ extraDeps; - srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} '' - cp ${proto} ${baseNameOf proto} - ${protobuf}/bin/protoc --plugin=${protoLibs.goProto.protoc-gen-go.gopkg}/bin/protoc-gen-go \ - --go_out=plugins=grpc,import_path=${baseNameOf path}:. ${baseNameOf proto} - mv *.pb.go $out - ''); - }; - - # Build a Go library out of the specified gRPC definition. - grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc.gopkg ]; }); - -in { - # Only the high-level builder functions are exposed, but made - # overrideable. - program = makeOverridable program; - package = makeOverridable package; - proto = makeOverridable proto; - grpc = makeOverridable grpc; - external = makeOverridable external; -} diff --git a/example/default.nix b/example/default.nix deleted file mode 100644 index 5abed1fbbcb5..000000000000 --- a/example/default.nix +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2019 Google LLC. -# SPDX-License-Identifier: Apache-2.0 - -# This file provides examples for how to use the various builder -# functions provided by `buildGo`. -# -# The features used in the example are not exhaustive, but should give -# users a quick introduction to how to use buildGo. - -let - buildGo = import ../buildGo.nix {}; - - # Example use of buildGo.package, which creates an importable Go - # package from the specified source files. - examplePackage = buildGo.package { - name = "example"; - srcs = [ - ./lib.go - ]; - }; - - # Example use of buildGo.proto, which generates a Go library from a - # Protobuf definition file. - exampleProto = buildGo.proto { - name = "exampleproto"; - proto = ./thing.proto; - }; - - # Example use of buildGo.program, which builds an executable using - # the specified name and dependencies (which in turn must have been - # created via buildGo.package etc.) -in buildGo.program { - name = "example"; - - srcs = [ - ./main.go - ]; - - deps = [ - examplePackage - exampleProto - ]; - - x_defs = { - "main.Flag" = "successfully"; - }; -} diff --git a/example/lib.go b/example/lib.go deleted file mode 100644 index 8a61370e994c..000000000000 --- a/example/lib.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2019 Google LLC. -// SPDX-License-Identifier: Apache-2.0 - -package example - -// UUID returns a totally random, carefully chosen UUID -func UUID() string { - return "3640932f-ad40-4bc9-b45d-f504a0f5910a" -} diff --git a/example/main.go b/example/main.go deleted file mode 100644 index bbcedbff8726..000000000000 --- a/example/main.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2019 Google LLC. -// SPDX-License-Identifier: Apache-2.0 -// -// Package main provides a tiny example program for the Bazel-style -// Nix build system for Go. - -package main - -import ( - "example" - "exampleproto" - "fmt" -) - -var Flag string = "unsuccessfully" - -func main() { - thing := exampleproto.Thing{ - Id: example.UUID(), - KindOfThing: "test thing", - } - - fmt.Printf("The thing is a %s with ID %q\n", thing.Id, thing.KindOfThing) - fmt.Printf("The flag has been %s set\n", Flag) -} diff --git a/example/thing.proto b/example/thing.proto deleted file mode 100644 index 0cb34124dfb9..000000000000 --- a/example/thing.proto +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2019 Google LLC. -// SPDX-License-Identifier: Apache-2.0 - -syntax = "proto3"; -package exampleProto; - -message Thing { - string id = 1; - string kind_of_thing = 2; -} diff --git a/external/default.nix b/external/default.nix deleted file mode 100644 index 6e1dce11140b..000000000000 --- a/external/default.nix +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2019 Google LLC. -# SPDX-License-Identifier: Apache-2.0 -{ pkgs, program, package }: - -let - inherit (builtins) - elemAt - foldl' - fromJSON - head - length - listToAttrs - readFile - replaceStrings - tail - throw; - - inherit (pkgs) lib runCommand go jq ripgrep; - - pathToName = p: replaceStrings ["/"] ["_"] (toString p); - - # Collect all non-vendored dependencies from the Go standard library - # into a file that can be used to filter them out when processing - # dependencies. - stdlibPackages = runCommand "stdlib-pkgs.json" {} '' - export GOPATH=/dev/null - ${go}/bin/go list all | \ - ${ripgrep}/bin/rg -v 'vendor' | \ - ${jq}/bin/jq -R '.' | \ - ${jq}/bin/jq -c -s 'map({key: ., value: true}) | from_entries' \ - > $out - ''; - - analyser = program { - name = "analyser"; - - srcs = [ - ./main.go - ]; - - x_defs = { - "main.stdlibList" = "${stdlibPackages}"; - }; - }; - - mkset = path: value: - if path == [] then { gopkg = value; } - else { "${head path}" = mkset (tail path) value; }; - - last = l: elemAt l ((length l) - 1); - - toPackage = self: src: path: depMap: entry: - let - localDeps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) ( - throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'" - ) self) entry.localDeps; - - foreignDeps = map (d: lib.attrByPath [ d ] ( - throw "missing foreign dependency '${d}' in '${path}'" - ) depMap) entry.foreignDeps; - - args = { - srcs = map (f: src + ("/" + f)) entry.files; - deps = localDeps ++ foreignDeps; - }; - - libArgs = args // { - name = pathToName entry.name; - path = lib.concatStringsSep "/" ([ path ] ++ entry.locator); - sfiles = map (f: src + ("/" + f)) entry.sfiles; - }; - - binArgs = args // { - name = (last ((lib.splitString "/" path) ++ entry.locator)); - }; - in if entry.isCommand then (program binArgs) else (package libArgs); - -in { src, path, deps ? [] }: let - # Build a map of dependencies (from their import paths to their - # derivation) so that they can be conditionally imported only in - # sub-packages that require them. - depMap = listToAttrs (map (d: { - name = d.goImportPath; - value = d; - }) deps); - - name = pathToName path; - analysisOutput = runCommand "${name}-structure.json" {} '' - ${analyser}/bin/analyser -path ${path} -source ${src} > $out - ''; - analysis = fromJSON (readFile analysisOutput); -in lib.fix(self: foldl' lib.recursiveUpdate {} ( - map (entry: mkset entry.locator (toPackage self src path depMap entry)) analysis -)) diff --git a/external/main.go b/external/main.go deleted file mode 100644 index aa4a813d32bd..000000000000 --- a/external/main.go +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2019 Google LLC. -// SPDX-License-Identifier: Apache-2.0 - -// This tool analyses external (i.e. not built with `buildGo.nix`) Go -// packages to determine a build plan that Nix can import. -package main - -import ( - "encoding/json" - "flag" - "fmt" - "go/build" - "io/ioutil" - "log" - "os" - "path" - "path/filepath" - "strings" -) - -// Path to a JSON file describing all standard library import paths. -// This file is generated and set here by Nix during the build -// process. -var stdlibList string - -// pkg describes a single Go package within the specified source -// directory. -// -// Return information includes the local (relative from project root) -// and external (none-stdlib) dependencies of this package. -type pkg struct { - Name string `json:"name"` - Locator []string `json:"locator"` - Files []string `json:"files"` - SFiles []string `json:"sfiles"` - LocalDeps [][]string `json:"localDeps"` - ForeignDeps []string `json:"foreignDeps"` - IsCommand bool `json:"isCommand"` -} - -// findGoDirs returns a filepath.WalkFunc that identifies all -// directories that contain Go source code in a certain tree. -func findGoDirs(at string) ([]string, error) { - dirSet := make(map[string]bool) - - err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error { - name := info.Name() - // Skip folders that are guaranteed to not be relevant - if info.IsDir() && (name == "testdata" || name == ".git") { - return filepath.SkipDir - } - - // If the current file is a Go file, then the directory is popped - // (i.e. marked as a Go directory). - if !info.IsDir() && strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go") { - dirSet[filepath.Dir(path)] = true - } - - return nil - }) - - if err != nil { - return nil, err - } - - goDirs := []string{} - for k, _ := range dirSet { - goDirs = append(goDirs, k) - } - - return goDirs, nil -} - -// analysePackage loads and analyses the imports of a single Go -// package, returning the data that is required by the Nix code to -// generate a derivation for this package. -func analysePackage(root, source, importpath string, stdlib map[string]bool) (pkg, error) { - ctx := build.Default - ctx.CgoEnabled = false - - p, err := ctx.ImportDir(source, build.IgnoreVendor) - if err != nil { - return pkg{}, err - } - - local := [][]string{} - foreign := []string{} - - for _, i := range p.Imports { - if stdlib[i] { - continue - } - - if i == importpath { - local = append(local, []string{}) - } else if strings.HasPrefix(i, importpath) { - local = append(local, strings.Split(strings.TrimPrefix(i, importpath+"/"), "/")) - } else { - foreign = append(foreign, i) - } - } - - prefix := strings.TrimPrefix(source, root+"/") - - locator := []string{} - if len(prefix) != len(source) { - locator = strings.Split(prefix, "/") - } else { - // Otherwise, the locator is empty since its the root package and - // no prefix should be added to files. - prefix = "" - } - - files := []string{} - for _, f := range p.GoFiles { - files = append(files, path.Join(prefix, f)) - } - - sfiles := []string{} - for _, f := range p.SFiles { - sfiles = append(sfiles, path.Join(prefix, f)) - } - - return pkg{ - Name: path.Join(importpath, prefix), - Locator: locator, - Files: files, - SFiles: sfiles, - LocalDeps: local, - ForeignDeps: foreign, - IsCommand: p.IsCommand(), - }, nil -} - -func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) { - f, err := ioutil.ReadFile(from) - if err != nil { - return - } - - err = json.Unmarshal(f, &pkgs) - return -} - -func main() { - source := flag.String("source", "", "path to directory with sources to process") - path := flag.String("path", "", "import path for the package") - - flag.Parse() - - if *source == "" { - log.Fatalf("-source flag must be specified") - } - - stdlibPkgs, err := loadStdlibPkgs(stdlibList) - if err != nil { - log.Fatalf("failed to load standard library index from %q: %s\n", stdlibList, err) - } - - goDirs, err := findGoDirs(*source) - if err != nil { - log.Fatalf("failed to walk source directory '%s': %s\n", source, err) - } - - all := []pkg{} - for _, d := range goDirs { - analysed, err := analysePackage(*source, d, *path, stdlibPkgs) - - // If the Go source analysis returned "no buildable Go files", - // that directory should be skipped. - // - // This might be due to `+build` flags on the platform and other - // reasons (such as test files). - if _, ok := err.(*build.NoGoError); ok { - continue - } - - if err != nil { - log.Fatalf("failed to analyse package at %q: %s", d, err) - } - all = append(all, analysed) - } - - j, _ := json.Marshal(all) - fmt.Println(string(j)) -} diff --git a/overrides/buildGo/README.md b/overrides/buildGo/README.md new file mode 100644 index 000000000000..e84ede663bf8 --- /dev/null +++ b/overrides/buildGo/README.md @@ -0,0 +1,140 @@ +buildGo.nix +=========== + +This is an alternative [Nix][] build system for [Go][]. It supports building Go +libraries and programs, and even automatically generating Protobuf & gRPC +libraries. + +*Note:* This will probably end up being folded into [Nixery][]. + +## Background + +Most language-specific Nix tooling outsources the build to existing +language-specific build tooling, which essentially means that Nix ends up being +a wrapper around all sorts of external build systems. + +However, systems like [Bazel][] take an alternative approach in which the +compiler is invoked directly and the composition of programs and libraries stays +within a single homogeneous build system. + +Users don't need to learn per-language build systems and especially for +companies with large monorepo-setups ([like Google][]) this has huge +productivity impact. + +This project is an attempt to prove that Nix can be used in a similar style to +build software directly, rather than shelling out to other build systems. + +## Example + +Given a program layout like this: + +``` +. +├── lib <-- some library component +│   ├── bar.go +│   └── foo.go +├── api.proto <-- gRPC API definition +├── main.go <-- program implementation +└── default.nix <-- build instructions +``` + +The contents of `default.nix` could look like this: + +```nix +{ buildGo }: + +let + api = buildGo.grpc { + name = "someapi"; + proto = ./api.proto; + }; + + lib = buildGo.package { + name = "somelib"; + srcs = [ + ./lib/bar.go + ./lib/foo.go + ]; + }; +in buildGo.program { + name = "my-program"; + deps = [ api lib ]; + + srcs = [ + ./main.go + ]; +} +``` + +(If you don't know how to read Nix, check out [nix-1p][]) + +## Usage + +`buildGo` exposes five different functions: + +* `buildGo.program`: Build a Go binary out of the specified source files. + + | parameter | type | use | required? | + |-----------|-------------------------|------------------------------------------------|-----------| + | `name` | `string` | Name of the program (and resulting executable) | yes | + | `srcs` | `list` | List of paths to source files | yes | + | `deps` | `list` | List of dependencies (i.e. other Go libraries) | no | + | `x_defs` | `attrs` | Attribute set of linker vars (i.e. `-X`-flags) | no | + +* `buildGo.package`: Build a Go library out of the specified source files. + + | parameter | type | use | required? | + |-----------|--------------|------------------------------------------------|-----------| + | `name` | `string` | Name of the library (and resulting executable) | yes | + | `srcs` | `list` | List of paths to source files | yes | + | `deps` | `list` | List of dependencies (i.e. other Go libraries) | no | + | `path` | `string` | Go import path for the resulting library | no | + +* `buildGo.external`: Build an externally defined Go library or program. + + This function performs analysis on the supplied source code (which + can use the standard Go tooling layout) and creates a tree of all + the packages contained within. + + This exists for compatibility with external libraries that were not + defined using buildGo. + + | parameter | type | use | required? | + |-----------|----------------|-----------------------------------------------|-----------| + | `path` | `string` | Go import path for the resulting package | yes | + | `src` | `path` | Path to the source **directory** | yes | + | `deps` | `list` | List of dependencies (i.e. other Go packages) | no | + + For some examples of how `buildGo.external` is used, check out + [`proto.nix`](./proto.nix). + +* `buildGo.proto`: Build a Go library out of the specified Protobuf definition. + + | parameter | type | use | required? | + |-------------|-------------|--------------------------------------------------|-----------| + | `name` | `string` | Name for the resulting library | yes | + | `proto` | `path` | Path to the Protobuf definition file | yes | + | `path` | `string` | Import path for the resulting Go library | no | + | `extraDeps` | `list` | Additional Go dependencies to add to the library | no | + +* `buildGo.grpc`: Build a Go library out of the specified gRPC definition. + + The parameters are identical to `buildGo.proto`. + +## Current status + +This project is work-in-progress. Crucially it is lacking the following features: + +* feature flag parity with Bazel's Go rules +* documentation building +* test execution + +There are still some open questions around how to structure some of those +features in Nix. + +[Nix]: https://nixos.org/nix/ +[Go]: https://golang.org/ +[Nixery]: https://github.com/google/nixery +[Bazel]: https://bazel.build/ +[like Google]: https://ai.google/research/pubs/pub45424 +[nix-1p]: https://github.com/tazjin/nix-1p diff --git a/overrides/buildGo/default.nix b/overrides/buildGo/default.nix new file mode 100644 index 000000000000..140cbf2d9d16 --- /dev/null +++ b/overrides/buildGo/default.nix @@ -0,0 +1,128 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 +# +# buildGo provides Nix functions to build Go packages in the style of Bazel's +# rules_go. + +{ pkgs ? import {} +, ... }: + +let + inherit (builtins) + attrNames + baseNameOf + dirOf + elemAt + filter + listToAttrs + map + match + readDir + replaceStrings + toString; + + inherit (pkgs) lib go runCommand fetchFromGitHub protobuf symlinkJoin; + + # Helpers for low-level Go compiler invocations + spaceOut = lib.concatStringsSep " "; + + includeDepSrc = dep: "-I ${dep}"; + includeSources = deps: spaceOut (map includeDepSrc deps); + + includeDepLib = dep: "-L ${dep}"; + includeLibs = deps: spaceOut (map includeDepLib deps); + + srcBasename = src: elemAt (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src)) 1; + srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}"; + srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs); + + allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps))); + + xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); + + pathToName = p: replaceStrings ["/"] ["_"] (toString p); + + # Add an `overrideGo` attribute to a function result that works + # similar to `overrideAttrs`, but is used specifically for the + # arguments passed to Go builders. + makeOverridable = f: orig: (f orig) // { + overrideGo = new: makeOverridable f (orig // (new orig)); + }; + + # High-level build functions + + # Build a Go program out of the specified files and dependencies. + program = { name, srcs, deps ? [], x_defs ? {} }: + let uniqueDeps = allDeps deps; + in runCommand name {} '' + ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs} + mkdir -p $out/bin + ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a + ''; + + # Build a Go library assembled out of the specified files. + # + # This outputs both the sources and compiled binary, as both are + # needed when downstream packages depend on it. + package = { name, srcs, deps ? [], path ? name, sfiles ? [] }: + let + uniqueDeps = allDeps deps; + + # The build steps below need to be executed conditionally for Go + # assembly if the analyser detected any *.s files. + # + # This is required for several popular packages (e.g. x/sys). + ifAsm = do: if sfiles == [] then "" else do; + asmBuild = ifAsm '' + ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles} + ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles} + ''; + asmLink = ifAsm "-symabis ./symabis -asmhdr $out/go_asm.h"; + asmPack = ifAsm '' + ${go}/bin/go tool pack r $out/${path}.a ./asm.o + ''; + in (runCommand "golib-${name}" {} '' + mkdir -p $out/${path} + ${srcList path (map (s: "${s}") srcs)} + ${asmBuild} + ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} + ${asmPack} + '') // { goDeps = uniqueDeps; goImportPath = path; }; + + # Build a tree of Go libraries out of an external Go source + # directory that follows the standard Go layout and was not built + # with buildGo.nix. + # + # The derivation for each actual package will reside in an attribute + # named "gopkg", and an attribute named "gobin" for binaries. + external = import ./external { inherit pkgs program package; }; + + # Import support libraries needed for protobuf & gRPC support + protoLibs = import ./proto.nix { + inherit external; + }; + + # Build a Go library out of the specified protobuf definition. + proto = { name, proto, path ? name, extraDeps ? [] }: (makeOverridable package) { + inherit name path; + deps = [ protoLibs.goProto.proto.gopkg ] ++ extraDeps; + srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} '' + cp ${proto} ${baseNameOf proto} + ${protobuf}/bin/protoc --plugin=${protoLibs.goProto.protoc-gen-go.gopkg}/bin/protoc-gen-go \ + --go_out=plugins=grpc,import_path=${baseNameOf path}:. ${baseNameOf proto} + mv *.pb.go $out + ''); + }; + + # Build a Go library out of the specified gRPC definition. + grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc.gopkg ]; }); + +in { + # Only the high-level builder functions are exposed, but made + # overrideable. + program = makeOverridable program; + package = makeOverridable package; + proto = makeOverridable proto; + grpc = makeOverridable grpc; + external = makeOverridable external; +} diff --git a/overrides/buildGo/example/default.nix b/overrides/buildGo/example/default.nix new file mode 100644 index 000000000000..5abed1fbbcb5 --- /dev/null +++ b/overrides/buildGo/example/default.nix @@ -0,0 +1,47 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 + +# This file provides examples for how to use the various builder +# functions provided by `buildGo`. +# +# The features used in the example are not exhaustive, but should give +# users a quick introduction to how to use buildGo. + +let + buildGo = import ../buildGo.nix {}; + + # Example use of buildGo.package, which creates an importable Go + # package from the specified source files. + examplePackage = buildGo.package { + name = "example"; + srcs = [ + ./lib.go + ]; + }; + + # Example use of buildGo.proto, which generates a Go library from a + # Protobuf definition file. + exampleProto = buildGo.proto { + name = "exampleproto"; + proto = ./thing.proto; + }; + + # Example use of buildGo.program, which builds an executable using + # the specified name and dependencies (which in turn must have been + # created via buildGo.package etc.) +in buildGo.program { + name = "example"; + + srcs = [ + ./main.go + ]; + + deps = [ + examplePackage + exampleProto + ]; + + x_defs = { + "main.Flag" = "successfully"; + }; +} diff --git a/overrides/buildGo/example/lib.go b/overrides/buildGo/example/lib.go new file mode 100644 index 000000000000..8a61370e994c --- /dev/null +++ b/overrides/buildGo/example/lib.go @@ -0,0 +1,9 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 + +package example + +// UUID returns a totally random, carefully chosen UUID +func UUID() string { + return "3640932f-ad40-4bc9-b45d-f504a0f5910a" +} diff --git a/overrides/buildGo/example/main.go b/overrides/buildGo/example/main.go new file mode 100644 index 000000000000..bbcedbff8726 --- /dev/null +++ b/overrides/buildGo/example/main.go @@ -0,0 +1,25 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 +// +// Package main provides a tiny example program for the Bazel-style +// Nix build system for Go. + +package main + +import ( + "example" + "exampleproto" + "fmt" +) + +var Flag string = "unsuccessfully" + +func main() { + thing := exampleproto.Thing{ + Id: example.UUID(), + KindOfThing: "test thing", + } + + fmt.Printf("The thing is a %s with ID %q\n", thing.Id, thing.KindOfThing) + fmt.Printf("The flag has been %s set\n", Flag) +} diff --git a/overrides/buildGo/example/thing.proto b/overrides/buildGo/example/thing.proto new file mode 100644 index 000000000000..0cb34124dfb9 --- /dev/null +++ b/overrides/buildGo/example/thing.proto @@ -0,0 +1,10 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; +package exampleProto; + +message Thing { + string id = 1; + string kind_of_thing = 2; +} diff --git a/overrides/buildGo/external/default.nix b/overrides/buildGo/external/default.nix new file mode 100644 index 000000000000..6e1dce11140b --- /dev/null +++ b/overrides/buildGo/external/default.nix @@ -0,0 +1,94 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 +{ pkgs, program, package }: + +let + inherit (builtins) + elemAt + foldl' + fromJSON + head + length + listToAttrs + readFile + replaceStrings + tail + throw; + + inherit (pkgs) lib runCommand go jq ripgrep; + + pathToName = p: replaceStrings ["/"] ["_"] (toString p); + + # Collect all non-vendored dependencies from the Go standard library + # into a file that can be used to filter them out when processing + # dependencies. + stdlibPackages = runCommand "stdlib-pkgs.json" {} '' + export GOPATH=/dev/null + ${go}/bin/go list all | \ + ${ripgrep}/bin/rg -v 'vendor' | \ + ${jq}/bin/jq -R '.' | \ + ${jq}/bin/jq -c -s 'map({key: ., value: true}) | from_entries' \ + > $out + ''; + + analyser = program { + name = "analyser"; + + srcs = [ + ./main.go + ]; + + x_defs = { + "main.stdlibList" = "${stdlibPackages}"; + }; + }; + + mkset = path: value: + if path == [] then { gopkg = value; } + else { "${head path}" = mkset (tail path) value; }; + + last = l: elemAt l ((length l) - 1); + + toPackage = self: src: path: depMap: entry: + let + localDeps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) ( + throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'" + ) self) entry.localDeps; + + foreignDeps = map (d: lib.attrByPath [ d ] ( + throw "missing foreign dependency '${d}' in '${path}'" + ) depMap) entry.foreignDeps; + + args = { + srcs = map (f: src + ("/" + f)) entry.files; + deps = localDeps ++ foreignDeps; + }; + + libArgs = args // { + name = pathToName entry.name; + path = lib.concatStringsSep "/" ([ path ] ++ entry.locator); + sfiles = map (f: src + ("/" + f)) entry.sfiles; + }; + + binArgs = args // { + name = (last ((lib.splitString "/" path) ++ entry.locator)); + }; + in if entry.isCommand then (program binArgs) else (package libArgs); + +in { src, path, deps ? [] }: let + # Build a map of dependencies (from their import paths to their + # derivation) so that they can be conditionally imported only in + # sub-packages that require them. + depMap = listToAttrs (map (d: { + name = d.goImportPath; + value = d; + }) deps); + + name = pathToName path; + analysisOutput = runCommand "${name}-structure.json" {} '' + ${analyser}/bin/analyser -path ${path} -source ${src} > $out + ''; + analysis = fromJSON (readFile analysisOutput); +in lib.fix(self: foldl' lib.recursiveUpdate {} ( + map (entry: mkset entry.locator (toPackage self src path depMap entry)) analysis +)) diff --git a/overrides/buildGo/external/main.go b/overrides/buildGo/external/main.go new file mode 100644 index 000000000000..aa4a813d32bd --- /dev/null +++ b/overrides/buildGo/external/main.go @@ -0,0 +1,186 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 + +// This tool analyses external (i.e. not built with `buildGo.nix`) Go +// packages to determine a build plan that Nix can import. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "go/build" + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + "strings" +) + +// Path to a JSON file describing all standard library import paths. +// This file is generated and set here by Nix during the build +// process. +var stdlibList string + +// pkg describes a single Go package within the specified source +// directory. +// +// Return information includes the local (relative from project root) +// and external (none-stdlib) dependencies of this package. +type pkg struct { + Name string `json:"name"` + Locator []string `json:"locator"` + Files []string `json:"files"` + SFiles []string `json:"sfiles"` + LocalDeps [][]string `json:"localDeps"` + ForeignDeps []string `json:"foreignDeps"` + IsCommand bool `json:"isCommand"` +} + +// findGoDirs returns a filepath.WalkFunc that identifies all +// directories that contain Go source code in a certain tree. +func findGoDirs(at string) ([]string, error) { + dirSet := make(map[string]bool) + + err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error { + name := info.Name() + // Skip folders that are guaranteed to not be relevant + if info.IsDir() && (name == "testdata" || name == ".git") { + return filepath.SkipDir + } + + // If the current file is a Go file, then the directory is popped + // (i.e. marked as a Go directory). + if !info.IsDir() && strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go") { + dirSet[filepath.Dir(path)] = true + } + + return nil + }) + + if err != nil { + return nil, err + } + + goDirs := []string{} + for k, _ := range dirSet { + goDirs = append(goDirs, k) + } + + return goDirs, nil +} + +// analysePackage loads and analyses the imports of a single Go +// package, returning the data that is required by the Nix code to +// generate a derivation for this package. +func analysePackage(root, source, importpath string, stdlib map[string]bool) (pkg, error) { + ctx := build.Default + ctx.CgoEnabled = false + + p, err := ctx.ImportDir(source, build.IgnoreVendor) + if err != nil { + return pkg{}, err + } + + local := [][]string{} + foreign := []string{} + + for _, i := range p.Imports { + if stdlib[i] { + continue + } + + if i == importpath { + local = append(local, []string{}) + } else if strings.HasPrefix(i, importpath) { + local = append(local, strings.Split(strings.TrimPrefix(i, importpath+"/"), "/")) + } else { + foreign = append(foreign, i) + } + } + + prefix := strings.TrimPrefix(source, root+"/") + + locator := []string{} + if len(prefix) != len(source) { + locator = strings.Split(prefix, "/") + } else { + // Otherwise, the locator is empty since its the root package and + // no prefix should be added to files. + prefix = "" + } + + files := []string{} + for _, f := range p.GoFiles { + files = append(files, path.Join(prefix, f)) + } + + sfiles := []string{} + for _, f := range p.SFiles { + sfiles = append(sfiles, path.Join(prefix, f)) + } + + return pkg{ + Name: path.Join(importpath, prefix), + Locator: locator, + Files: files, + SFiles: sfiles, + LocalDeps: local, + ForeignDeps: foreign, + IsCommand: p.IsCommand(), + }, nil +} + +func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) { + f, err := ioutil.ReadFile(from) + if err != nil { + return + } + + err = json.Unmarshal(f, &pkgs) + return +} + +func main() { + source := flag.String("source", "", "path to directory with sources to process") + path := flag.String("path", "", "import path for the package") + + flag.Parse() + + if *source == "" { + log.Fatalf("-source flag must be specified") + } + + stdlibPkgs, err := loadStdlibPkgs(stdlibList) + if err != nil { + log.Fatalf("failed to load standard library index from %q: %s\n", stdlibList, err) + } + + goDirs, err := findGoDirs(*source) + if err != nil { + log.Fatalf("failed to walk source directory '%s': %s\n", source, err) + } + + all := []pkg{} + for _, d := range goDirs { + analysed, err := analysePackage(*source, d, *path, stdlibPkgs) + + // If the Go source analysis returned "no buildable Go files", + // that directory should be skipped. + // + // This might be due to `+build` flags on the platform and other + // reasons (such as test files). + if _, ok := err.(*build.NoGoError); ok { + continue + } + + if err != nil { + log.Fatalf("failed to analyse package at %q: %s", d, err) + } + all = append(all, analysed) + } + + j, _ := json.Marshal(all) + fmt.Println(string(j)) +} diff --git a/overrides/buildGo/proto.nix b/overrides/buildGo/proto.nix new file mode 100644 index 000000000000..2ece948ebd84 --- /dev/null +++ b/overrides/buildGo/proto.nix @@ -0,0 +1,84 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 +# +# This file provides derivations for the dependencies of a gRPC +# service in Go. + +{ external }: + +let + inherit (builtins) fetchGit map; +in rec { + goProto = external { + path = "github.com/golang/protobuf"; + src = fetchGit { + url = "https://github.com/golang/protobuf"; + rev = "ed6926b37a637426117ccab59282c3839528a700"; + }; + }; + + xnet = external { + path = "golang.org/x/net"; + + src = fetchGit { + url = "https://go.googlesource.com/net"; + rev = "ffdde105785063a81acd95bdf89ea53f6e0aac2d"; + }; + + deps = map (p: p.gopkg) [ + xtext.secure.bidirule + xtext.unicode.bidi + xtext.unicode.norm + ]; + }; + + xsys = external { + path = "golang.org/x/sys"; + src = fetchGit { + url = "https://go.googlesource.com/sys"; + rev = "bd437916bb0eb726b873ee8e9b2dcf212d32e2fd"; + }; + }; + + xtext = external { + path = "golang.org/x/text"; + src = fetchGit { + url = "https://go.googlesource.com/text"; + rev = "cbf43d21aaebfdfeb81d91a5f444d13a3046e686"; + }; + }; + + genproto = external { + path = "google.golang.org/genproto"; + src = fetchGit { + url = "https://github.com/google/go-genproto"; + rev = "83cc0476cb11ea0da33dacd4c6354ab192de6fe6"; + }; + + deps = with goProto; map (p: p.gopkg) [ + proto + ptypes.any + ]; + }; + + goGrpc = external { + path = "google.golang.org/grpc"; + deps = map (p: p.gopkg) ([ + xnet.trace + xnet.http2 + xsys.unix + xnet.http2.hpack + genproto.googleapis.rpc.status + ] ++ (with goProto; [ + proto + ptypes + ptypes.duration + ptypes.timestamp + ])); + + src = fetchGit { + url = "https://github.com/grpc/grpc-go"; + rev = "d8e3da36ac481ef00e510ca119f6b68177713689"; + }; + }; +} diff --git a/proto.nix b/proto.nix deleted file mode 100644 index 2ece948ebd84..000000000000 --- a/proto.nix +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2019 Google LLC. -# SPDX-License-Identifier: Apache-2.0 -# -# This file provides derivations for the dependencies of a gRPC -# service in Go. - -{ external }: - -let - inherit (builtins) fetchGit map; -in rec { - goProto = external { - path = "github.com/golang/protobuf"; - src = fetchGit { - url = "https://github.com/golang/protobuf"; - rev = "ed6926b37a637426117ccab59282c3839528a700"; - }; - }; - - xnet = external { - path = "golang.org/x/net"; - - src = fetchGit { - url = "https://go.googlesource.com/net"; - rev = "ffdde105785063a81acd95bdf89ea53f6e0aac2d"; - }; - - deps = map (p: p.gopkg) [ - xtext.secure.bidirule - xtext.unicode.bidi - xtext.unicode.norm - ]; - }; - - xsys = external { - path = "golang.org/x/sys"; - src = fetchGit { - url = "https://go.googlesource.com/sys"; - rev = "bd437916bb0eb726b873ee8e9b2dcf212d32e2fd"; - }; - }; - - xtext = external { - path = "golang.org/x/text"; - src = fetchGit { - url = "https://go.googlesource.com/text"; - rev = "cbf43d21aaebfdfeb81d91a5f444d13a3046e686"; - }; - }; - - genproto = external { - path = "google.golang.org/genproto"; - src = fetchGit { - url = "https://github.com/google/go-genproto"; - rev = "83cc0476cb11ea0da33dacd4c6354ab192de6fe6"; - }; - - deps = with goProto; map (p: p.gopkg) [ - proto - ptypes.any - ]; - }; - - goGrpc = external { - path = "google.golang.org/grpc"; - deps = map (p: p.gopkg) ([ - xnet.trace - xnet.http2 - xsys.unix - xnet.http2.hpack - genproto.googleapis.rpc.status - ] ++ (with goProto; [ - proto - ptypes - ptypes.duration - ptypes.timestamp - ])); - - src = fetchGit { - url = "https://github.com/grpc/grpc-go"; - rev = "d8e3da36ac481ef00e510ca119f6b68177713689"; - }; - }; -} -- cgit 1.4.1