From 78ca29dd23a327f5199ad263ea057d701ddefed6 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 21 Nov 2019 10:55:10 +0000 Subject: feat(buildGo): Introduce Bazel-style Nix builders for Go --- overrides/buildGo.nix | 169 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 overrides/buildGo.nix (limited to 'overrides/buildGo.nix') diff --git a/overrides/buildGo.nix b/overrides/buildGo.nix new file mode 100644 index 000000000000..e6531393db69 --- /dev/null +++ b/overrides/buildGo.nix @@ -0,0 +1,169 @@ +# buildGo provides Nix functions to build Go packages in the style of Bazel's +# rules_go. +# +# TODO(tazjin): Go through Bazel rules_go options and implement corresponding flags +# TODO(tazjin): Refactor to include /golang/protobuf/descriptor in goProto deps + +{ pkgs, ... }@args: + +let + inherit (builtins) + attrNames + baseNameOf + currentSystem + filter + isString + head + map + match + readDir + toFile + toPath + toString; + + inherit (pkgs) bash lib go runCommand fetchFromGitHub protobuf; + + # Helpers for low-level Go compiler invocations + spaceOut = lib.concatStringsSep " "; + + includeDepSrc = dep: "-I ${dep}"; + includeSources = deps: spaceOut (map includeDepSrc deps); + + includeDepLib = dep: "-L ${dep}"; + includeLibs = deps: spaceOut (map includeDepLib deps); + + srcBasename = src: head (filter isString (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src))); + srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}"; + srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs); + + isGoFile = f: (match ".*\.go" f) != null; + isGoTest = f: (match ".*_test\.go" f) != null; + goFileFilter = k: v: (v == "regular") && (isGoFile k) && (!isGoTest k); + goFilesIn = dir: + let files = readDir dir; + goFiles = filter (f: goFileFilter f files."${f}") (attrNames files); + in map (f: dir + "/" + f) goFiles; + + allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps))); + + # High-level build functions + + # Build a Go program out of the specified files and dependencies. + program = { name, srcs, deps ? [] }: + let uniqueDeps = allDeps deps; + in runCommand name {} '' + ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD ${includeSources uniqueDeps} ${spaceOut srcs} + mkdir -p $out/bin + ${go}/bin/go tool link -o $out/bin/${name} ${includeLibs uniqueDeps} ${name}.a + ''; + + # Build a Go library assembled out of the specified files. + # + # This outputs both the sources and compiled binary, as both are + # needed when downstream packages depend on it. + package = { name, srcs, deps ? [], path ? name }: + let uniqueDeps = allDeps deps; + in (runCommand "golib-${name}" {} '' + mkdir -p $out/${path} + ${srcList path (map (s: "${s}") srcs)} + ${go}/bin/go tool compile -o $out/${path}.a -trimpath=$PWD -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} + '') // { goDeps = uniqueDeps; }; + + # Build a Go library out of the specified proto/gRPC file. + proto = { name, proto, path ? name }: package { + inherit name path; + deps = [ goProto ]; + srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} '' + cp ${proto} ${baseNameOf proto} + ${protobuf}/bin/protoc --plugin=${protocGo}/bin/protoc-gen-go \ + --go_out=import_path=${baseNameOf path}:. ${baseNameOf proto} + mv *.pb.go $out + ''); + }; + + # Protobuf & gRPC integration requires these dependencies: + proto-go-src = fetchFromGitHub { + owner = "golang"; + repo = "protobuf"; + rev = "ed6926b37a637426117ccab59282c3839528a700"; + sha256 = "0fynqrim022x9xi2bivkw19npbz4316v4yr7mb677s9s36z4dc4h"; + }; + + goProto = + let + protobuf = package { + name = "protobuf"; + path = "github.com/golang/protobuf/proto"; + srcs = filter + (f: (match "(.*)/pointer_reflect.go" f) == null) + (goFilesIn (toPath "${proto-go-src}/proto")); + }; + type = name: package { + name = "ptypes-${name}"; + path = "github.com/golang/protobuf/ptypes/${name}"; + srcs = goFilesIn (toPath "${proto-go-src}/ptypes/${name}"); + deps = [ protobuf ]; + }; + # descriptor = package { + # name = "descriptor"; + # path = "github.com/golang/protobuf/descriptor"; + # deps = [ protobuf ]; + # srcs = goFilesIn (toPath "${proto-go-src}/descriptor"); + # }; + ptypes = package { + name = "ptypes"; + path = "github.com/golang/protobuf/ptypes"; + srcs = goFilesIn (toPath "${proto-go-src}/ptypes"); + deps = map type [ + "any" + "duration" + "empty" + "struct" + "timestamp" + "wrappers" + ]; + }; + in protobuf // { goDeps = allDeps (protobuf.goDeps ++ [ ptypes ]); }; + + protocGo = + let + descriptor = package { + name = "descriptor"; + path = "github.com/golang/protobuf/protoc-gen-go/descriptor"; + srcs = goFilesIn (toPath "${proto-go-src}/protoc-gen-go/descriptor"); + deps = [ goProto ]; + }; + plugin = package { + name = "plugin"; + path = "github.com/golang/protobuf/protoc-gen-go/plugin"; + srcs = goFilesIn (toPath "${proto-go-src}/protoc-gen-go/plugin"); + deps = [ descriptor ]; + }; + remap = package { + name = "remap"; + path = "github.com/golang/protobuf/protoc-gen-go/generator/internal/remap"; + srcs = goFilesIn (toPath "${proto-go-src}/protoc-gen-go/generator/internal/remap"); + }; + generator = package { + name = "generator"; + path = "github.com/golang/protobuf/protoc-gen-go/generator"; + srcs = goFilesIn (toPath "${proto-go-src}/protoc-gen-go/generator"); + deps = [ descriptor remap plugin ]; + }; + grpc = package { + name = "grpc"; + path = "github.com/golang/protobuf/protoc-gen-go/grpc"; + deps = [ generator ]; + srcs = goFilesIn (toPath "${proto-go-src}/protoc-gen-go/grpc"); + }; + in program { + name = "protoc-gen-go"; + deps = [ goProto grpc generator ]; + srcs = filter + (f: (match "(.*)/doc.go" f) == null) + (goFilesIn (toPath "${proto-go-src}/protoc-gen-go")); + }; +in { + # Only the high-level builder functions are exposed + inherit program package proto; +} -- cgit 1.4.1 From 5a89a0265b60ab4e354d0ce0f7cd0bad2753eebc Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 22 Nov 2019 16:14:14 +0000 Subject: refactor(buildGo): Minor cleanups of buildGo implementation --- overrides/buildGo.nix | 77 +++++++++++++++++---------------------------------- 1 file changed, 25 insertions(+), 52 deletions(-) (limited to 'overrides/buildGo.nix') diff --git a/overrides/buildGo.nix b/overrides/buildGo.nix index e6531393db69..8e915dda14a8 100644 --- a/overrides/buildGo.nix +++ b/overrides/buildGo.nix @@ -3,6 +3,7 @@ # # TODO(tazjin): Go through Bazel rules_go options and implement corresponding flags # TODO(tazjin): Refactor to include /golang/protobuf/descriptor in goProto deps +# TODO(tazjin): Find a way to expose documentation (esp. for generated stuff) { pkgs, ... }@args: @@ -10,16 +11,13 @@ let inherit (builtins) attrNames baseNameOf - currentSystem + elemAt filter - isString - head map match readDir - toFile - toPath - toString; + replaceStrings + toPath; inherit (pkgs) bash lib go runCommand fetchFromGitHub protobuf; @@ -66,17 +64,17 @@ let in (runCommand "golib-${name}" {} '' mkdir -p $out/${path} ${srcList path (map (s: "${s}") srcs)} - ${go}/bin/go tool compile -o $out/${path}.a -trimpath=$PWD -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} + ${go}/bin/go tool compile -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} '') // { goDeps = uniqueDeps; }; - # Build a Go library out of the specified proto/gRPC file. - proto = { name, proto, path ? name }: package { + # Build a Go library out of the specified protobuf definition. + proto = { name, proto, path ? name, protocFlags ? "", extraDeps ? [] }: package { inherit name path; - deps = [ goProto ]; + deps = [ goProto ] ++ extraDeps; srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} '' cp ${proto} ${baseNameOf proto} ${protobuf}/bin/protoc --plugin=${protocGo}/bin/protoc-gen-go \ - --go_out=import_path=${baseNameOf path}:. ${baseNameOf proto} + --go_out=${protocFlags}import_path=${baseNameOf path}:. ${baseNameOf proto} mv *.pb.go $out ''); }; @@ -89,27 +87,25 @@ let sha256 = "0fynqrim022x9xi2bivkw19npbz4316v4yr7mb677s9s36z4dc4h"; }; + protoPart = path: deps: package { + inherit deps; + name = replaceStrings ["/"] ["_"] path; + path = "github.com/golang/protobuf/${path}"; + srcs = goFilesIn (toPath "${proto-go-src}/${path}"); + }; + goProto = let protobuf = package { name = "protobuf"; path = "github.com/golang/protobuf/proto"; + # TODO(tazjin): How does this build toggle work? srcs = filter (f: (match "(.*)/pointer_reflect.go" f) == null) (goFilesIn (toPath "${proto-go-src}/proto")); }; - type = name: package { - name = "ptypes-${name}"; - path = "github.com/golang/protobuf/ptypes/${name}"; - srcs = goFilesIn (toPath "${proto-go-src}/ptypes/${name}"); - deps = [ protobuf ]; - }; - # descriptor = package { - # name = "descriptor"; - # path = "github.com/golang/protobuf/descriptor"; - # deps = [ protobuf ]; - # srcs = goFilesIn (toPath "${proto-go-src}/descriptor"); - # }; + type = name: protoPart "ptypes/${name}" [ protobuf ]; + descriptor = protoPart "descriptor" [ protobuf ]; ptypes = package { name = "ptypes"; path = "github.com/golang/protobuf/ptypes"; @@ -125,37 +121,14 @@ let }; in protobuf // { goDeps = allDeps (protobuf.goDeps ++ [ ptypes ]); }; + protocDescriptor = (protoPart "protoc-gen-go/descriptor" [ goProto ]); protocGo = let - descriptor = package { - name = "descriptor"; - path = "github.com/golang/protobuf/protoc-gen-go/descriptor"; - srcs = goFilesIn (toPath "${proto-go-src}/protoc-gen-go/descriptor"); - deps = [ goProto ]; - }; - plugin = package { - name = "plugin"; - path = "github.com/golang/protobuf/protoc-gen-go/plugin"; - srcs = goFilesIn (toPath "${proto-go-src}/protoc-gen-go/plugin"); - deps = [ descriptor ]; - }; - remap = package { - name = "remap"; - path = "github.com/golang/protobuf/protoc-gen-go/generator/internal/remap"; - srcs = goFilesIn (toPath "${proto-go-src}/protoc-gen-go/generator/internal/remap"); - }; - generator = package { - name = "generator"; - path = "github.com/golang/protobuf/protoc-gen-go/generator"; - srcs = goFilesIn (toPath "${proto-go-src}/protoc-gen-go/generator"); - deps = [ descriptor remap plugin ]; - }; - grpc = package { - name = "grpc"; - path = "github.com/golang/protobuf/protoc-gen-go/grpc"; - deps = [ generator ]; - srcs = goFilesIn (toPath "${proto-go-src}/protoc-gen-go/grpc"); - }; + generator = protoPart "protoc-gen-go/generator" [ + (protoPart "protoc-gen-go/generator/internal/remap" []) + (protoPart "protoc-gen-go/plugin" [ protocDescriptor ]) + ]; + grpc = protoPart "protoc-gen-go/grpc" [ generator ]; in program { name = "protoc-gen-go"; deps = [ goProto grpc generator ]; -- cgit 1.4.1 From 8710df08c8212d33a8b8d8bf22524fe290642842 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 22 Nov 2019 16:14:35 +0000 Subject: fix(buildGo): Ensure names of proto-generated files are correct --- overrides/buildGo.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'overrides/buildGo.nix') diff --git a/overrides/buildGo.nix b/overrides/buildGo.nix index 8e915dda14a8..7ba9000e183b 100644 --- a/overrides/buildGo.nix +++ b/overrides/buildGo.nix @@ -30,7 +30,7 @@ let includeDepLib = dep: "-L ${dep}"; includeLibs = deps: spaceOut (map includeDepLib deps); - srcBasename = src: head (filter isString (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src))); + srcBasename = src: elemAt (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src)) 1; srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}"; srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs); -- cgit 1.4.1 From 9ea0363e6f34a9f41b3c849359c97f06dbec2b9f Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 22 Nov 2019 16:14:52 +0000 Subject: fix(buildGo): Use stable build ID when linking outputs This is another step towards build reproducibility, which Go is traditionally very bad at. --- overrides/buildGo.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'overrides/buildGo.nix') diff --git a/overrides/buildGo.nix b/overrides/buildGo.nix index 7ba9000e183b..72d8c232f6e3 100644 --- a/overrides/buildGo.nix +++ b/overrides/buildGo.nix @@ -50,9 +50,9 @@ let program = { name, srcs, deps ? [] }: let uniqueDeps = allDeps deps; in runCommand name {} '' - ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD ${includeSources uniqueDeps} ${spaceOut srcs} + ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs} mkdir -p $out/bin - ${go}/bin/go tool link -o $out/bin/${name} ${includeLibs uniqueDeps} ${name}.a + ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${includeLibs uniqueDeps} ${name}.a ''; # Build a Go library assembled out of the specified files. -- cgit 1.4.1 From 580bd8862257eedeec7680b86590afcb524fa858 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Mon, 25 Nov 2019 15:34:59 +0000 Subject: feat(buildGo): Add x_defs support --- overrides/buildGo.nix | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'overrides/buildGo.nix') diff --git a/overrides/buildGo.nix b/overrides/buildGo.nix index 72d8c232f6e3..8e1349bf91a9 100644 --- a/overrides/buildGo.nix +++ b/overrides/buildGo.nix @@ -1,3 +1,6 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 +# # buildGo provides Nix functions to build Go packages in the style of Bazel's # rules_go. # @@ -5,7 +8,7 @@ # TODO(tazjin): Refactor to include /golang/protobuf/descriptor in goProto deps # TODO(tazjin): Find a way to expose documentation (esp. for generated stuff) -{ pkgs, ... }@args: +{ pkgs, ... }: let inherit (builtins) @@ -19,7 +22,7 @@ let replaceStrings toPath; - inherit (pkgs) bash lib go runCommand fetchFromGitHub protobuf; + inherit (pkgs) lib go runCommand fetchFromGitHub protobuf; # Helpers for low-level Go compiler invocations spaceOut = lib.concatStringsSep " "; @@ -44,15 +47,17 @@ let allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps))); + xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); + # High-level build functions # Build a Go program out of the specified files and dependencies. - program = { name, srcs, deps ? [] }: + program = { name, srcs, deps ? [], x_defs ? {} }: let uniqueDeps = allDeps deps; in runCommand name {} '' ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs} mkdir -p $out/bin - ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${includeLibs uniqueDeps} ${name}.a + ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a ''; # Build a Go library assembled out of the specified files. -- cgit 1.4.1 From 0bd085f5d6602ab75689156b92af04e7f83d69fd Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Wed, 27 Nov 2019 15:18:40 +0000 Subject: chore(overrides): Import buildGo.nix from external repository --- overrides/buildGo.nix | 151 ++------------------------------------------------ 1 file changed, 4 insertions(+), 147 deletions(-) (limited to 'overrides/buildGo.nix') diff --git a/overrides/buildGo.nix b/overrides/buildGo.nix index 8e1349bf91a9..cbccfcec976d 100644 --- a/overrides/buildGo.nix +++ b/overrides/buildGo.nix @@ -1,147 +1,4 @@ -# Copyright 2019 Google LLC. -# SPDX-License-Identifier: Apache-2.0 -# -# buildGo provides Nix functions to build Go packages in the style of Bazel's -# rules_go. -# -# TODO(tazjin): Go through Bazel rules_go options and implement corresponding flags -# TODO(tazjin): Refactor to include /golang/protobuf/descriptor in goProto deps -# TODO(tazjin): Find a way to expose documentation (esp. for generated stuff) - -{ pkgs, ... }: - -let - inherit (builtins) - attrNames - baseNameOf - elemAt - filter - map - match - readDir - replaceStrings - toPath; - - inherit (pkgs) lib go runCommand fetchFromGitHub protobuf; - - # Helpers for low-level Go compiler invocations - spaceOut = lib.concatStringsSep " "; - - includeDepSrc = dep: "-I ${dep}"; - includeSources = deps: spaceOut (map includeDepSrc deps); - - includeDepLib = dep: "-L ${dep}"; - includeLibs = deps: spaceOut (map includeDepLib deps); - - srcBasename = src: elemAt (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src)) 1; - srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}"; - srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs); - - isGoFile = f: (match ".*\.go" f) != null; - isGoTest = f: (match ".*_test\.go" f) != null; - goFileFilter = k: v: (v == "regular") && (isGoFile k) && (!isGoTest k); - goFilesIn = dir: - let files = readDir dir; - goFiles = filter (f: goFileFilter f files."${f}") (attrNames files); - in map (f: dir + "/" + f) goFiles; - - allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps))); - - xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); - - # High-level build functions - - # Build a Go program out of the specified files and dependencies. - program = { name, srcs, deps ? [], x_defs ? {} }: - let uniqueDeps = allDeps deps; - in runCommand name {} '' - ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs} - mkdir -p $out/bin - ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a - ''; - - # Build a Go library assembled out of the specified files. - # - # This outputs both the sources and compiled binary, as both are - # needed when downstream packages depend on it. - package = { name, srcs, deps ? [], path ? name }: - let uniqueDeps = allDeps deps; - in (runCommand "golib-${name}" {} '' - mkdir -p $out/${path} - ${srcList path (map (s: "${s}") srcs)} - ${go}/bin/go tool compile -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} - '') // { goDeps = uniqueDeps; }; - - # Build a Go library out of the specified protobuf definition. - proto = { name, proto, path ? name, protocFlags ? "", extraDeps ? [] }: package { - inherit name path; - deps = [ goProto ] ++ extraDeps; - srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} '' - cp ${proto} ${baseNameOf proto} - ${protobuf}/bin/protoc --plugin=${protocGo}/bin/protoc-gen-go \ - --go_out=${protocFlags}import_path=${baseNameOf path}:. ${baseNameOf proto} - mv *.pb.go $out - ''); - }; - - # Protobuf & gRPC integration requires these dependencies: - proto-go-src = fetchFromGitHub { - owner = "golang"; - repo = "protobuf"; - rev = "ed6926b37a637426117ccab59282c3839528a700"; - sha256 = "0fynqrim022x9xi2bivkw19npbz4316v4yr7mb677s9s36z4dc4h"; - }; - - protoPart = path: deps: package { - inherit deps; - name = replaceStrings ["/"] ["_"] path; - path = "github.com/golang/protobuf/${path}"; - srcs = goFilesIn (toPath "${proto-go-src}/${path}"); - }; - - goProto = - let - protobuf = package { - name = "protobuf"; - path = "github.com/golang/protobuf/proto"; - # TODO(tazjin): How does this build toggle work? - srcs = filter - (f: (match "(.*)/pointer_reflect.go" f) == null) - (goFilesIn (toPath "${proto-go-src}/proto")); - }; - type = name: protoPart "ptypes/${name}" [ protobuf ]; - descriptor = protoPart "descriptor" [ protobuf ]; - ptypes = package { - name = "ptypes"; - path = "github.com/golang/protobuf/ptypes"; - srcs = goFilesIn (toPath "${proto-go-src}/ptypes"); - deps = map type [ - "any" - "duration" - "empty" - "struct" - "timestamp" - "wrappers" - ]; - }; - in protobuf // { goDeps = allDeps (protobuf.goDeps ++ [ ptypes ]); }; - - protocDescriptor = (protoPart "protoc-gen-go/descriptor" [ goProto ]); - protocGo = - let - generator = protoPart "protoc-gen-go/generator" [ - (protoPart "protoc-gen-go/generator/internal/remap" []) - (protoPart "protoc-gen-go/plugin" [ protocDescriptor ]) - ]; - grpc = protoPart "protoc-gen-go/grpc" [ generator ]; - in program { - name = "protoc-gen-go"; - deps = [ goProto grpc generator ]; - srcs = filter - (f: (match "(.*)/doc.go" f) == null) - (goFilesIn (toPath "${proto-go-src}/protoc-gen-go")); - }; -in { - # Only the high-level builder functions are exposed - inherit program package proto; -} +import "${builtins.fetchGit { + url = "https://github.com/tazjin/buildGo.nix"; + rev = "28e587b348a8aaa7af00a004c05286af9d35ca9a"; +}}/buildGo.nix" -- cgit 1.4.1