merge(feat/besadii): Integrate besadii at //ops/besadii r/569

This tool supersedes the previous //ops/sync-gcsr, which no longer
exists.

Besadii runs as a git post-receive hook and triggers builds for the
depot on sourcehut. Builds are only triggered for branches, not for
tags.

The sourcehut token is read from /etc/secrets where it has been
manually placed on camden.

Besadii was a hutt kajidic, and also part of the Hutt Clan.

6 files changed, 210 insertions, 240 deletions

diff --git a/ci-builds.nix b/ci-builds.nix
index 624f4893b9..e9abcdb610 100644
--- a/ci-builds.nix
+++ b/ci-builds.nix
@@ -14,13 +14,13 @@ with (import ./default.nix {}); [
   nix.buildLisp.example
   nix.yants.tests
   ops."posix_mq.rs"
+  ops.besadii
   ops.journaldriver
   ops.kms_pass
   ops.kontemplate
   ops.mq_cli
   ops.nixos.camdenSystem
   ops.nixos.nuggetSystem
-  ops.sync-gcsr
   third_party.cgit
   third_party.git
   third_party.guile
diff --git a/ops/besadii/default.nix b/ops/besadii/default.nix
new file mode 100644
index 0000000000..31f2705d73
--- /dev/null
+++ b/ops/besadii/default.nix
@@ -0,0 +1,12 @@
+# This program is used as a git post-update hook to trigger builds on
+# sourcehut.
+{ depot, ... }:
+
+depot.buildGo.program {
+  name = "besadii";
+  srcs = [ ./main.go ];
+
+  x_defs = {
+    "main.gitBin" = "${depot.third_party.git}/bin/git";
+  };
+}
diff --git a/ops/besadii/main.go b/ops/besadii/main.go
new file mode 100644
index 0000000000..460eddf265
--- /dev/null
+++ b/ops/besadii/main.go
@@ -0,0 +1,197 @@
+// Copyright 2019 Google LLC.
+// SPDX-License-Identifier: Apache-2.0
+//
+// besadii is a small CLI tool that triggers depot builds on
+// builds.sr.ht
+//
+// It is designed to run as a post-update git hook on the server
+// hosting the depot.
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log/syslog"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+var gitBin = "git"
+
+// Represents an updated reference as passed to besadii by git
+//
+// https://git-scm.com/docs/githooks#pre-receive
+type refUpdate struct {
+	name string
+	old  string
+	new  string
+}
+
+// Represents a builds.sr.ht build object as described on
+// https://man.sr.ht/builds.sr.ht/api.md
+type Build struct {
+	Manifest string   `json:"manifest"`
+	Note     string   `json:"note"`
+	Tags     []string `json:"tags"`
+}
+
+// Represents a build trigger object as described on <the docs for
+// this are currently down>
+type Trigger struct {
+	Action    string `json:"action"`
+	Condition string `json:"condition"`
+	To        string `json:"to"`
+}
+
+// Represents a build manifest for sourcehut.
+type Manifest struct {
+	Image    string                `json:"image"`
+	Sources  []string              `json:"sources"`
+	Secrets  []string              `json:"secrets"`
+	Tasks    [](map[string]string) `json:"tasks"`
+	Triggers []Trigger             `json:"triggers"`
+}
+
+func prepareManifest(commit string) string {
+	m := Manifest{
+		Image:   "nixos/latest",
+		Sources: []string{"https://git.camden.tazj.in/"},
+
+		// secret for cachix/tazjin
+		Secrets: []string{"f7f02546-4d95-44f7-a98e-d61fdded8b5b"},
+
+		Tasks: [](map[string]string){
+			{"setup": `# sourcehut does not censor secrets in builds, hence this hack:
+echo -n 'export CACHIX_SIGNING_KEY=' >> ~/.buildenv
+cat ~/.cachix-tazjin >> ~/.buildenv
+nix-env -iA third_party.cachix -f git.tazj.in
+cachix use tazjin
+cd git.tazj.in
+git checkout ` + commit},
+
+			{"build": `cd git.tazj.in
+nix-build ci-builds.nix > built-paths`},
+
+			{"cache": `cd git.tazj.in
+cat built-paths | cachix push tazjin`},
+		},
+
+		Triggers: []Trigger{
+			Trigger{Action: "email", Condition: "failure", To: "mail@tazj.in"},
+		},
+	}
+
+	j, _ := json.Marshal(m)
+	return string(j)
+}
+
+// Trigger a build of a given branch & commit on builds.sr.ht
+func triggerBuild(log *syslog.Writer, token, branch, commit string) {
+	build := Build{
+		Manifest: prepareManifest(commit),
+		Note:     fmt.Sprintf("Build of 'master' at '%s'", commit),
+		Tags: []string{
+			// my branch names tend to contain slashes, which are not valid
+			// identifiers in sourcehut.
+			"depot", strings.ReplaceAll(branch, "/", "_"),
+		},
+	}
+
+	body, _ := json.Marshal(build)
+	reader := ioutil.NopCloser(bytes.NewReader(body))
+
+	req, err := http.NewRequest("POST", "https://builds.sr.ht/api/jobs", reader)
+	if err != nil {
+		log.Err(fmt.Sprintf("failed to create an HTTP request: %s", err))
+		os.Exit(1)
+	}
+
+	req.Header.Add("Authorization", "token "+token)
+	req.Header.Add("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		// This might indicate a temporary error on the sourcehut side, do
+		// not fail the whole program.
+		log.Err(fmt.Sprintf("failed to send builds.sr.ht request:", err))
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		respBody, _ := ioutil.ReadAll(resp.Body)
+		log.Err(fmt.Sprintf("received non-success response from builds.sr.ht: %s (%v)", respBody, resp.Status))
+	} else {
+		fmt.Fprintf(log, "triggered builds.sr.ht job for branch '%s' at commit '%s'", branch, commit)
+	}
+}
+
+func parseRefUpdates() ([]refUpdate, error) {
+	var updates []refUpdate
+
+	scanner := bufio.NewScanner(os.Stdin)
+	for scanner.Scan() {
+		line := scanner.Text()
+		fragments := strings.Split(line, " ")
+
+		if len(fragments) != 3 {
+			return nil, fmt.Errorf("invalid ref update: '%s'", line)
+		}
+
+		if !strings.HasPrefix(fragments[2], "refs/heads/") {
+			continue
+		}
+
+		updates = append(updates, refUpdate{
+			old:  fragments[0],
+			new:  fragments[1],
+			name: strings.TrimPrefix(fragments[2], "refs/heads/"),
+		})
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	return updates, nil
+}
+
+func main() {
+	log, err := syslog.New(syslog.LOG_INFO|syslog.LOG_USER, "besadii")
+	if err != nil {
+		fmt.Printf("failed to open syslog: %s\n", err)
+		os.Exit(1)
+	}
+
+	// Before triggering builds, it is important that git
+	// update-server-info is run so that cgit correctly serves the
+	// repository.
+	err = exec.Command(gitBin, "update-server-info").Run()
+	if err != nil {
+		log.Alert("failed to run 'git update-server-info' for depot!")
+		os.Exit(1)
+	}
+
+	token, err := ioutil.ReadFile("/etc/secrets/srht-token")
+	if err != nil {
+		log.Alert("sourcehot token could not be read")
+		os.Exit(1)
+	}
+
+	updates, err := parseRefUpdates()
+	if err != nil {
+		log.Err(fmt.Sprintf("could not parse updated refs:", err))
+		os.Exit(1)
+	}
+
+	fmt.Fprintf(log, "triggering builds for %v refs", len(updates))
+
+	for _, update := range updates {
+		triggerBuild(log, string(token), update.name, update.new)
+	}
+}
diff --git a/ops/sync-gcsr/default.nix b/ops/sync-gcsr/default.nix
deleted file mode 100644
index ea5d0594a8..0000000000
--- a/ops/sync-gcsr/default.nix
+++ /dev/null
@@ -1,14 +0,0 @@
-{ depot, ... }:
-
-depot.nix.buildGo.program {
-  name = "sync-gcsr";
-  srcs = [ ./main.go ];
-
-  deps = with depot.third_party; map (p: p.gopkg) [
-    gopkgs."gopkg.in".src-d.go-git
-  ];
-
-  x_defs = {
-    "main.BuildManifest" = "${./manifest.yaml}";
-  };
-}
diff --git a/ops/sync-gcsr/main.go b/ops/sync-gcsr/main.go
deleted file mode 100644
index 62c24a92ce..0000000000
--- a/ops/sync-gcsr/main.go
+++ /dev/null
@@ -1,202 +0,0 @@
-// Copyright 2019 Google LLC.
-// SPDX-License-Identifier: Apache-2.0
-//
-// sync-gcsr implements a small utility that periodically mirrors a
-// remote Google Cloud Source Repository to a local file path.
-//
-// This utility is also responsible for triggering depot builds on
-// builds.sr.ht if a change is detected on the master branch.
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"time"
-	"bytes"
-
-	git "gopkg.in/src-d/go-git.v4"
-	"gopkg.in/src-d/go-git.v4/plumbing"
-	githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
-)
-
-// Path to the build manifest, added by Nix at compile time.
-var BuildManifest string
-
-// Represents a builds.sr.ht build object as described on
-// https://man.sr.ht/builds.sr.ht/api.md
-type Build struct {
-	Manifest string   `json:"manifest"`
-	Note     string   `json:"note"`
-	Tags     []string `json:"tags"`
-}
-
-func EnvOr(key, def string) string {
-	v := os.Getenv(key)
-	if v == "" {
-		return def
-	}
-
-	return v
-}
-
-// Trigger a build of master on builds.sr.ht
-func triggerBuild(commit string) {
-	manifest, err := ioutil.ReadFile(BuildManifest)
-	if err != nil {
-		log.Fatalln("[ERROR] failed to read sr.ht build manifest:", err)
-	}
-
-	build := Build{
-		Manifest: string(manifest),
-		Note:     fmt.Sprintf("Build of 'master' at '%s'", commit),
-		Tags: []string{
-			"depot", "master",
-		},
-	}
-
-	body, _ := json.Marshal(build)
-	reader := ioutil.NopCloser(bytes.NewReader(body))
-
-	req, err := http.NewRequest("POST", "https://builds.sr.ht/api/jobs", reader)
-	if err != nil {
-		log.Fatalln("[ERROR] failed to create an HTTP request:", err)
-	}
-
-	req.Header.Add("Authorization", fmt.Sprintf("token %s", os.Getenv("SRHT_TOKEN")))
-	req.Header.Add("Content-Type", "application/json")
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		// This might indicate a temporary error on the SourceHut side, do
-		// not fail the whole program.
-		log.Println("failed to send builds.sr.ht request:", err)
-		return
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		respBody, err := ioutil.ReadAll(resp.Body)
-		log.Printf("received non-success response from builds.sr.ht: %s (%v)[%s]", respBody, resp.Status, err)
-	} else {
-		log.Println("triggered builds.sr.ht job for commit", commit)
-	}
-}
-
-// ensure that all remote branches exist locally & are up to date.
-func updateBranches(auth *githttp.BasicAuth, repo *git.Repository) error {
-	origin, err := repo.Remote("origin")
-	if err != nil {
-		return err
-	}
-
-	refs, err := origin.List(&git.ListOptions{
-		Auth: auth,
-	})
-	if err != nil {
-		return err
-	}
-
-	for _, ref := range refs {
-		if !ref.Name().IsBranch() || ref.Type() != plumbing.HashReference {
-			continue
-		}
-
-		name := plumbing.NewBranchReferenceName(ref.Name().Short())
-
-		if current, err := repo.Storer.Reference(name); err == nil {
-			// Determine whether the reference has changed to skip
-			// unnecessary modifications.
-			if current.Hash() == ref.Hash() {
-				continue
-			}
-		}
-
-		branch := plumbing.NewHashReference(name, ref.Hash())
-
-		err := repo.Storer.SetReference(branch)
-		if err != nil {
-			return err
-		}
-
-		if ref.Name().Short() == "master" {
-			go triggerBuild(ref.Hash().String())
-		}
-
-		log.Println("Updated branch", ref.Name().String())
-	}
-
-	return nil
-}
-
-func updateRepo(auth *githttp.BasicAuth, repo *git.Repository, opts *git.FetchOptions) error {
-	err := repo.Fetch(opts)
-
-	if err == git.NoErrAlreadyUpToDate {
-		// nothing to do ...
-		return nil
-	} else if err != nil {
-		return err
-	}
-
-	log.Println("Fetched updates from remote, updating local branches")
-	return updateBranches(auth, repo)
-}
-
-func cloneRepo(dest, project, repo string, auth *githttp.BasicAuth) (*git.Repository, error) {
-	var cloneOpts = git.CloneOptions{
-		Auth: auth,
-		URL:  fmt.Sprintf("https://source.developers.google.com/p/%s/r/%s", project, repo),
-	}
-
-	handle, err := git.PlainClone(dest, true, &cloneOpts)
-
-	if err == git.ErrRepositoryAlreadyExists {
-		handle, err = git.PlainOpen(dest)
-	}
-
-	return handle, updateBranches(auth, handle)
-}
-
-func main() {
-	dest := EnvOr("SYNC_DEST", "/git/depot")
-	project := EnvOr("SYNC_PROJECT", "tazjins-infrastructure")
-	repo := EnvOr("SYNC_REPO", "depot")
-	user := os.Getenv("SYNC_USER")
-	pass := os.Getenv("SYNC_PASS")
-
-	log.Printf("Syncing repository '%s/%s' to destination '%s'", project, repo, dest)
-
-	var auth *githttp.BasicAuth
-	if user != "" && pass != "" {
-		auth = &githttp.BasicAuth{
-			Username: user,
-			Password: pass,
-		}
-		log.Println("Enabling basic authentication as user", user)
-	}
-
-	handle, err := cloneRepo(dest, project, repo, auth)
-
-	if err != nil {
-		log.Fatalf("Failed to clone repository: %s", err)
-	} else {
-		log.Println("Initiating update loop")
-	}
-
-	fetchOpts := git.FetchOptions{
-		Auth:  auth,
-		Force: true,
-	}
-
-	for {
-		if err = updateRepo(auth, handle, &fetchOpts); err != nil {
-			log.Fatalf("Failed to pull updated repository: %s", err)
-		}
-
-		time.Sleep(10 * time.Second)
-	}
-}
diff --git a/ops/sync-gcsr/manifest.yaml b/ops/sync-gcsr/manifest.yaml
deleted file mode 100644
index 3016c2ca57..0000000000
--- a/ops/sync-gcsr/manifest.yaml
+++ /dev/null
@@ -1,23 +0,0 @@
-image: nixos/latest
-sources:
-  - https://git.tazj.in/
-secrets:
-  # cachix/tazjin
-  - f7f02546-4d95-44f7-a98e-d61fdded8b5b
-tasks:
-  - setup: |
-      # sourcehut does not censor secrets in builds, hence this hack:
-      echo -n 'export CACHIX_SIGNING_KEY=' > cachix-preamble
-      cat cachix-preamble ~/.cachix-tazjin >> ~/.buildenv
-      nix-env -iA third_party.cachix -f git.tazj.in
-      cachix use tazjin
-  - build: |
-      cd git.tazj.in
-      nix-build ci-builds.nix > built-paths
-  - cache: |
-      cd git.tazj.in
-      cat built-paths | cachix push tazjin
-triggers:
-  - action: email
-    condition: failure
-    to: mail@tazj.in