From 6edf69da4bb2a5f18b2a44bb3fdf14ba7ea7e901 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 26 Jun 2020 22:22:01 +0100 Subject: feat(besadii): Trigger separate builds for build target sets In CL/570 we split up the build targets into different buckets, with the idea that this should help us avoid the disk space issues on Sourcehut. This commit changes Besadii to read the list of target sets from a file and trigger a separate build for each one of them. Change-Id: If280fda5f40cd130c534c40911072e47c2d8f2be Reviewed-on: https://cl.tvl.fyi/c/depot/+/608 Reviewed-by: lukegb --- ops/besadii/default.nix | 10 ++++++-- ops/besadii/main.go2 | 68 +++++++++++++++++++++++++++++++++++++------------ 2 files changed, 60 insertions(+), 18 deletions(-) (limited to 'ops') diff --git a/ops/besadii/default.nix b/ops/besadii/default.nix index d9f29dfc515c..91d6bba6ac52 100644 --- a/ops/besadii/default.nix +++ b/ops/besadii/default.nix @@ -1,8 +1,14 @@ # This program is used as a git post-update hook to trigger builds on # sourcehut. -{ depot, ... }: +{ ciBuilds, depot, ... }: -depot.nix.buildTypedGo.program { +let + inherit (builtins) toFile toJSON; +in depot.nix.buildTypedGo.program { name = "besadii"; srcs = [ ./main.go2 ]; + + x_defs = { + "main.TargetList" = toFile "ci-targets.json" (toJSON ciBuilds.__evaluatable); + }; } diff --git a/ops/besadii/main.go2 b/ops/besadii/main.go2 index cdc6405b43cf..ae5159435110 100644 --- a/ops/besadii/main.go2 +++ b/ops/besadii/main.go2 @@ -23,6 +23,11 @@ import ( var branchPrefix = "refs/heads/" +// TargetList is a path to the file containing the list of build +// targets in the depot, as a JSON array of strings. This is populated +// by the Nix build. +var TargetList string + // Represents an updated branch, as passed to besadii by Gerrit. // // https://gerrit.googlesource.com/plugins/hooks/+/HEAD/src/main/resources/Documentation/hooks.md#ref_updated @@ -58,27 +63,27 @@ type Manifest struct { Triggers []Trigger `json:"triggers"` } -func prepareManifest(commit string) string { +func prepareManifest(commit, target string) string { m := Manifest{ Image: "nixos/latest", - Sources: []string{"https://code.tvl.fyi/"}, + Sources: []string{"https://cl.tvl.fyi/depot"}, - // secret for cachix/tazjin + // ID of the secret for cachix/tazjin Secrets: []string{"f7f02546-4d95-44f7-a98e-d61fdded8b5b"}, Tasks: [](map[string]string){ {"setup": `# sourcehut does not censor secrets in builds, hence this hack: echo -n 'export CACHIX_SIGNING_KEY=' >> ~/.buildenv cat ~/.cachix-tazjin >> ~/.buildenv -nix-env -iA third_party.cachix -f code.tvl.fyi +nix-env -iA third_party.cachix -f depot cachix use tazjin -cd code.tvl.fyi +cd depot git checkout ` + commit}, - {"build": `cd code.tvl.fyi -nix-build ci-builds.nix > built-paths`}, + {"build": fmt.Sprintf(`cd depot +nix-build -A ciBuilds.%s > built-paths`, target)}, - {"cache": `cd code.tvl.fyi + {"cache": `cd depot cat built-paths | cachix push tazjin`}, }, @@ -92,14 +97,16 @@ cat built-paths | cachix push tazjin`}, } // Trigger a build of a given branch & commit on builds.sr.ht -func triggerBuild(log *syslog.Writer, token string, update *branchUpdate) error { +func triggerBuild(log *syslog.Writer, token string, update *branchUpdate, target string) error { build := Build{ - Manifest: prepareManifest(update.commit), - Note: fmt.Sprintf("build of %q at %q, submitted by %q", update.branch, update.commit, update.submitter), + Manifest: prepareManifest(update.commit, target), + Note: fmt.Sprintf("build of target set %q on %q at %q, submitted by %q", target, update.branch, update.commit, update.submitter), Tags: []string{ - // my branch names tend to contain slashes, which are not valid + "depot", + // our branch names tend to contain slashes, which are not valid // identifiers in sourcehut. - "depot", strings.ReplaceAll(update.branch, "/", "_"), + strings.ReplaceAll(update.branch, "/", "_"), + target, }, } @@ -190,6 +197,26 @@ func branchUpdateFromFlags() (*branchUpdate, error) { return &update, nil } +func loadBuildTargets() ([]string, error) { + if TargetList == "" { + return nil, fmt.Errorf("target list file was not set!") + } + + var targets []string + raw, err := ioutil.ReadFile(TargetList) + + if err != nil { + return nil, fmt.Errorf("failed to read build targets: %w", err) + } + + err = json.Unmarshal(raw, &targets) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal build targets: %w", err) + } + + return targets, nil +} + func main() { log, err := syslog.New(syslog.LOG_INFO|syslog.LOG_USER, "besadii") if err != nil { @@ -197,6 +224,12 @@ func main() { os.Exit(1) } + targets, err := loadBuildTargets() + if err != nil { + log.Alert(fmt.Sprintf("failed to load build targets: %s", err)) + os.Exit(1) + } + srhtToken, err := ioutil.ReadFile("/etc/secrets/srht-token") if err != nil { log.Alert("sourcehut token could not be read") @@ -219,13 +252,16 @@ func main() { os.Exit(0) } - err = triggerBuild(log, string(srhtToken), update) - if err != nil { - log.Err(fmt.Sprintf("failed to trigger sr.ht build: %s", err)) + for _, target := range targets { + err = triggerBuild(log, string(srhtToken), update, target) + if err != nil { + log.Err(fmt.Sprintf("failed to trigger sr.ht build: %s", err)) + } } err = triggerIndexUpdate(string(sourcegraphToken)) if err != nil { log.Err(fmt.Sprintf("failed to trigger sourcegraph index update: %s", err)) } + log.Info("triggered sourcegraph index update") } -- cgit 1.4.1