1 files changed, 558 insertions, 0 deletions
diff --git a/ops/besadii/main.go b/ops/besadii/main.go
new file mode 100644
index 000000000000..e22dbb1e484e
--- /dev/null
+++ b/ops/besadii/main.go
@@ -0,0 +1,558 @@
+// Copyright 2019-2020 Google LLC.
+// SPDX-License-Identifier: Apache-2.0
+//
+// besadii is a small CLI tool that is invoked as a hook by various
+// programs to cause CI-related actions.
+//
+// It supports the following modes & operations:
+//
+// Gerrit (ref-updated) hook:
+// - Trigger Buildkite CI builds
+// - Trigger SourceGraph repository index updates
+//
+// Buildkite (post-command) hook:
+// - Submit CL verification status back to Gerrit
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log/syslog"
+	"net/http"
+	"net/mail"
+	"os"
+	"os/user"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// Regular expression to extract change ID out of a URL
+var changeIdRegexp = regexp.MustCompile(`^.*/(\d+)$`)
+
+// Regular expression to check if gerritChangeName valid. The
+// limitation could be what is allowed for a git branch name. For now
+// we want to have a stricter limitation for readability and ease of
+// use.
+var gerritChangeNameRegexp = `^[a-z0-9]+$`
+var gerritChangeNameCheck = regexp.MustCompile(gerritChangeNameRegexp)
+
+// besadii configuration file structure
+type config struct {
+	// Required configuration for Buildkite<>Gerrit monorepo
+	// integration.
+	Repository       string `json:"repository"`
+	Branch           string `json:"branch"`
+	GerritUrl        string `json:"gerritUrl"`
+	GerritUser       string `json:"gerritUser"`
+	GerritPassword   string `json:"gerritPassword"`
+	GerritLabel      string `json:"gerritLabel"`
+	BuildkiteOrg     string `json:"buildkiteOrg"`
+	BuildkiteProject string `json:"buildkiteProject"`
+	BuildkiteToken   string `json:"buildkiteToken"`
+	GerritChangeName string `json:"gerritChangeName"`
+
+	// Optional configuration for Sourcegraph trigger updates.
+	SourcegraphUrl   string `json:"sourcegraphUrl"`
+	SourcegraphToken string `json:"sourcegraphToken"`
+}
+
+// buildTrigger represents the information passed to besadii when it
+// is invoked as a Gerrit hook.
+//
+// https://gerrit.googlesource.com/plugins/hooks/+/HEAD/src/main/resources/Documentation/hooks.md
+type buildTrigger struct {
+	project string
+	ref     string
+	commit  string
+	author  string
+	email   string
+
+	changeId string
+	patchset string
+}
+
+type Author struct {
+	Name  string `json:"name"`
+	Email string `json:"email"`
+}
+
+// Build is the representation of a Buildkite build as described on
+// https://buildkite.com/docs/apis/rest-api/builds#create-a-build
+type Build struct {
+	Commit string            `json:"commit"`
+	Branch string            `json:"branch"`
+	Author Author            `json:"author"`
+	Env    map[string]string `json:"env"`
+}
+
+// BuildResponse is the representation of Buildkite's success response
+// after triggering a build. This has many fields, but we only need
+// one of them.
+type buildResponse struct {
+	WebUrl string `json:"web_url"`
+}
+
+// reviewInput is a struct representing the data submitted to Gerrit
+// to post a review on a CL.
+//
+// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input
+type reviewInput struct {
+	Message                        string         `json:"message"`
+	Labels                         map[string]int `json:"labels,omitempty"`
+	OmitDuplicateComments          bool           `json:"omit_duplicate_comments"`
+	IgnoreDefaultAttentionSetRules bool           `json:"ignore_default_attention_set_rules"`
+	Tag                            string         `json:"tag"`
+	Notify                         string         `json:"notify,omitempty"`
+}
+
+func defaultConfigLocation() (string, error) {
+	usr, err := user.Current()
+	if err != nil {
+		return "", fmt.Errorf("failed to get current user: %w", err)
+	}
+
+	return path.Join(usr.HomeDir, "besadii.json"), nil
+}
+
+func loadConfig() (*config, error) {
+	configPath := os.Getenv("BESADII_CONFIG")
+
+	if configPath == "" {
+		var err error
+		configPath, err = defaultConfigLocation()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get config location: %w", err)
+		}
+	}
+
+	configJson, err := ioutil.ReadFile(configPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load besadii config: %w", err)
+	}
+
+	var cfg config
+	err = json.Unmarshal(configJson, &cfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal besadii config: %w", err)
+	}
+
+	// The default Gerrit label to set is 'Verified', unless specified otherwise.
+	if cfg.GerritLabel == "" {
+		cfg.GerritLabel = "Verified"
+	}
+
+	// The default text referring to a Gerrit Change in BuildKite.
+	if cfg.GerritChangeName == "" {
+		cfg.GerritChangeName = "cl"
+	}
+	if !gerritChangeNameCheck.MatchString(cfg.GerritChangeName) {
+		return nil, fmt.Errorf("invalid 'gerritChangeName': %s", cfg.GerritChangeName)
+	}
+
+	// Rudimentary config validation logic
+	if cfg.SourcegraphUrl != "" && cfg.SourcegraphToken == "" {
+		return nil, fmt.Errorf("'SourcegraphToken' must be set if 'SourcegraphUrl' is set")
+	}
+
+	if cfg.Repository == "" || cfg.Branch == "" {
+		return nil, fmt.Errorf("missing repository configuration (required: repository, branch)")
+	}
+
+	if cfg.GerritUrl == "" || cfg.GerritUser == "" || cfg.GerritPassword == "" {
+		return nil, fmt.Errorf("missing Gerrit configuration (required: gerritUrl, gerritUser, gerritPassword)")
+	}
+
+	if cfg.BuildkiteOrg == "" || cfg.BuildkiteProject == "" || cfg.BuildkiteToken == "" {
+		return nil, fmt.Errorf("mising Buildkite configuration (required: buildkiteOrg, buildkiteProject, buildkiteToken)")
+	}
+
+	return &cfg, nil
+}
+
+// linkToChange creates the full link to a change's patchset in Gerrit
+func linkToChange(cfg *config, changeId, patchset string) string {
+	return fmt.Sprintf("%s/c/%s/+/%s/%s", cfg.GerritUrl, cfg.Repository, changeId, patchset)
+}
+
+// updateGerrit posts a comment on a Gerrit CL to indicate the current build status.
+func updateGerrit(cfg *config, review reviewInput, changeId, patchset string) {
+	body, _ := json.Marshal(review)
+	reader := ioutil.NopCloser(bytes.NewReader(body))
+
+	url := fmt.Sprintf("%s/a/changes/%s/revisions/%s/review", cfg.GerritUrl, changeId, patchset)
+	req, err := http.NewRequest("POST", url, reader)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create an HTTP request: %w", err)
+		os.Exit(1)
+	}
+
+	req.SetBasicAuth(cfg.GerritUser, cfg.GerritPassword)
+	req.Header.Add("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		fmt.Errorf("failed to update %s on %s: %w", cfg.GerritChangeName, cfg.GerritUrl, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		respBody, _ := ioutil.ReadAll(resp.Body)
+		fmt.Fprintf(os.Stderr, "received non-success response from Gerrit: %s (%v)", respBody, resp.Status)
+	} else {
+		fmt.Printf("Added CI status comment on %s", linkToChange(cfg, changeId, patchset))
+	}
+}
+
+// Trigger a build of a given branch & commit on Buildkite
+func triggerBuild(cfg *config, log *syslog.Writer, trigger *buildTrigger) error {
+	env := make(map[string]string)
+	branch := trigger.ref
+
+	// Pass information about the originating Gerrit change to the
+	// build, if it is for a patchset.
+	//
+	// This information is later used by besadii when invoked by Gerrit
+	// to communicate the build status back to Gerrit.
+	headBuild := true
+	if trigger.changeId != "" && trigger.patchset != "" {
+		env["GERRIT_CHANGE_URL"] = linkToChange(cfg, trigger.changeId, trigger.patchset)
+		env["GERRIT_CHANGE_ID"] = trigger.changeId
+		env["GERRIT_PATCHSET"] = trigger.patchset
+		headBuild = false
+
+		// The branch doesn't have to be a real ref (it's just used to
+		// group builds), so make it the identifier for the CL.
+		branch = fmt.Sprintf("%s/%v", cfg.GerritChangeName, strings.Split(trigger.ref, "/")[3])
+	}
+
+	build := Build{
+		Commit: trigger.commit,
+		Branch: branch,
+		Env:    env,
+		Author: Author{
+			Name:  trigger.author,
+			Email: trigger.email,
+		},
+	}
+
+	body, _ := json.Marshal(build)
+	reader := ioutil.NopCloser(bytes.NewReader(body))
+
+	bkUrl := fmt.Sprintf("https://api.buildkite.com/v2/organizations/%s/pipelines/%s/builds", cfg.BuildkiteOrg, cfg.BuildkiteProject)
+	req, err := http.NewRequest("POST", bkUrl, reader)
+	if err != nil {
+		return fmt.Errorf("failed to create an HTTP request: %w", err)
+	}
+
+	req.Header.Add("Authorization", "Bearer "+cfg.BuildkiteToken)
+	req.Header.Add("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		// This might indicate a temporary error on the Buildkite side.
+		return fmt.Errorf("failed to send Buildkite request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	respBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed to read Buildkite response body: %w", err)
+	}
+
+	if resp.StatusCode != http.StatusCreated {
+		return fmt.Errorf("received non-success response from Buildkite: %s (%v)", respBody, resp.Status)
+	}
+
+	var buildResp buildResponse
+	err = json.Unmarshal(respBody, &buildResp)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal build response: %w", err)
+	}
+
+	fmt.Fprintf(log, "triggered build for ref %q at commit %q: %s", trigger.ref, trigger.commit, buildResp.WebUrl)
+
+	// For builds of the HEAD branch there is nothing else to do
+	if headBuild {
+		return nil
+	}
+
+	// Report the status back to the Gerrit CL so that users can click
+	// through to the running build.
+	msg := fmt.Sprintf("Started build for patchset #%s on: %s", trigger.patchset, buildResp.WebUrl)
+	review := reviewInput{
+		Message:               msg,
+		OmitDuplicateComments: true,
+		Tag:                   "autogenerated:buildkite~trigger",
+
+		// Do not update the attention set for this comment.
+		IgnoreDefaultAttentionSetRules: true,
+
+		Notify: "NONE",
+	}
+	updateGerrit(cfg, review, trigger.changeId, trigger.patchset)
+
+	return nil
+}
+
+// Trigger a Sourcegraph repository index update.
+//
+// https://docs.sourcegraph.com/admin/repo/webhooks
+func triggerIndexUpdate(cfg *config, log *syslog.Writer) error {
+	req, err := http.NewRequest("POST", cfg.SourcegraphUrl, nil)
+	if err != nil {
+		return err
+	}
+
+	req.Header.Add("Authorization", "token "+cfg.SourcegraphToken)
+
+	_, err = http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to trigger Sourcegraph index update: %w", err)
+	}
+
+	log.Info("triggered sourcegraph index update")
+	return nil
+}
+
+// Gerrit passes more flags than we want, but Rob Pike decided[0] in
+// 2013 that the Go art project will not allow users to ignore flags
+// because he "doesn't like it". This function allows users to ignore
+// flags.
+//
+// [0]: https://github.com/golang/go/issues/6112#issuecomment-66083768
+func ignoreFlags(ignore []string) {
+	for _, f := range ignore {
+		flag.String(f, "", "flag to ignore")
+	}
+}
+
+// Extract the username & email from Gerrit's uploader flag and set it
+// on the trigger struct, for display in Buildkite.
+func extractChangeUploader(uploader string, trigger *buildTrigger) error {
+	// Gerrit passes the uploader in another extra layer of quotes.
+	uploader, err := strconv.Unquote(uploader)
+	if err != nil {
+		return fmt.Errorf("failed to unquote email - forgot quotes on manual invocation?: %w", err)
+	}
+
+	// Extract the uploader username & email from the input passed by
+	// Gerrit (in RFC 5322 format).
+	addr, err := mail.ParseAddress(uploader)
+	if err != nil {
+		return fmt.Errorf("invalid change uploader (%s): %w", uploader, err)
+	}
+
+	trigger.author = addr.Name
+	trigger.email = addr.Address
+
+	return nil
+}
+
+// Extract the buildtrigger struct out of the flags passed to besadii
+// when invoked as Gerrit's 'patchset-created' hook. This hook is used
+// for triggering CI on in-progress CLs.
+func buildTriggerFromPatchsetCreated(cfg *config) (*buildTrigger, error) {
+	// Information that needs to be returned
+	var trigger buildTrigger
+
+	// Information that is only needed for parsing
+	var targetBranch, changeUrl, uploader, kind string
+
+	flag.StringVar(&trigger.project, "project", "", "Gerrit project")
+	flag.StringVar(&trigger.commit, "commit", "", "commit hash")
+	flag.StringVar(&trigger.patchset, "patchset", "", "patchset ID")
+
+	flag.StringVar(&targetBranch, "branch", "", "CL target branch")
+	flag.StringVar(&changeUrl, "change-url", "", "HTTPS URL of change")
+	flag.StringVar(&uploader, "uploader", "", "Change uploader name & email")
+	flag.StringVar(&kind, "kind", "", "Kind of patchset")
+
+	// patchset-created also passes various flags which we don't need.
+	ignoreFlags([]string{"topic", "change", "uploader-username", "change-owner", "change-owner-username"})
+
+	flag.Parse()
+
+	// Ignore patchsets which do not contain code changes
+	if kind == "NO_CODE_CHANGE" || kind == "NO_CHANGE" {
+		return nil, nil
+	}
+
+	// Parse username & email
+	err := extractChangeUploader(uploader, &trigger)
+	if err != nil {
+		return nil, err
+	}
+
+	// If the patchset is not for the HEAD branch of the monorepo, then
+	// we can ignore it. It might be some other kind of change
+	// (refs/meta/config or Gerrit-internal), but it is not an error.
+	if trigger.project != cfg.Repository || targetBranch != cfg.Branch {
+		return nil, nil
+	}
+
+	// Change ID is not directly passed in the numeric format, so we
+	// need to extract it out of the URL
+	matches := changeIdRegexp.FindStringSubmatch(changeUrl)
+	trigger.changeId = matches[1]
+
+	// Construct the CL ref from which the build should happen.
+	changeId, _ := strconv.Atoi(trigger.changeId)
+	trigger.ref = fmt.Sprintf(
+		"refs/changes/%02d/%s/%s",
+		changeId%100, trigger.changeId, trigger.patchset,
+	)
+
+	return &trigger, nil
+}
+
+// Extract the buildtrigger struct out of the flags passed to besadii
+// when invoked as Gerrit's 'change-merged' hook. This hook is used
+// for triggering HEAD builds after change submission.
+func buildTriggerFromChangeMerged(cfg *config) (*buildTrigger, error) {
+	// Information that needs to be returned
+	var trigger buildTrigger
+
+	// Information that is only needed for parsing
+	var targetBranch, submitter string
+
+	flag.StringVar(&trigger.project, "project", "", "Gerrit project")
+	flag.StringVar(&trigger.commit, "commit", "", "Commit hash")
+	flag.StringVar(&submitter, "submitter", "", "Submitter email & username")
+	flag.StringVar(&targetBranch, "branch", "", "CL target branch")
+
+	// Ignore extra flags passed by change-merged
+	ignoreFlags([]string{"change", "topic", "change-url", "submitter-username", "newrev", "change-owner", "change-owner-username"})
+
+	flag.Parse()
+
+	// Parse username & email
+	err := extractChangeUploader(submitter, &trigger)
+	if err != nil {
+		return nil, err
+	}
+
+	// If the patchset is not for the HEAD branch of the monorepo, then
+	// we can ignore it.
+	if trigger.project != cfg.Repository || targetBranch != cfg.Branch {
+		return nil, nil
+	}
+
+	trigger.ref = "refs/heads/" + targetBranch
+
+	return &trigger, nil
+}
+
+func gerritHookMain(cfg *config, log *syslog.Writer, trigger *buildTrigger) {
+	if trigger == nil {
+		// The hook was not for something we care about.
+		os.Exit(0)
+	}
+
+	err := triggerBuild(cfg, log, trigger)
+
+	if err != nil {
+		log.Err(fmt.Sprintf("failed to trigger Buildkite build: %s", err))
+	}
+
+	if cfg.SourcegraphUrl != "" && trigger.ref == "refs/heads/canon" {
+		err = triggerIndexUpdate(cfg, log)
+		if err != nil {
+			log.Err(fmt.Sprintf("failed to trigger sourcegraph index update: %s", err))
+		}
+	}
+}
+
+func postCommandMain(cfg *config) {
+	changeId := os.Getenv("GERRIT_CHANGE_ID")
+	patchset := os.Getenv("GERRIT_PATCHSET")
+
+	if changeId == "" || patchset == "" {
+		// If these variables are unset, but the hook was invoked, the
+		// build was most likely for a branch and not for a CL - no status
+		// needs to be reported back to Gerrit!
+		fmt.Println("This isn't a %s build, nothing to do. Have a nice day!", cfg.GerritChangeName)
+		return
+	}
+
+	if os.Getenv("BUILDKITE_LABEL") != ":duck:" {
+		// this is not the build stage, don't do anything.
+		return
+	}
+
+	var vote int
+	var verb string
+	var notify string
+
+	if os.Getenv("BUILDKITE_COMMAND_EXIT_STATUS") == "0" {
+		vote = 1 // automation passed: +1 in Gerrit
+		verb = "passed"
+		notify = "NONE"
+	} else {
+		vote = -1
+		verb = "failed"
+		notify = "OWNER"
+	}
+
+	msg := fmt.Sprintf("Build of patchset %s %s: %s", patchset, verb, os.Getenv("BUILDKITE_BUILD_URL"))
+	review := reviewInput{
+		Message:               msg,
+		OmitDuplicateComments: true,
+		Labels: map[string]int{
+			cfg.GerritLabel: vote,
+		},
+
+		// Update the attention set if we are failing this patchset.
+		IgnoreDefaultAttentionSetRules: vote == 1,
+
+		Tag: "autogenerated:buildkite~result",
+
+		Notify: notify,
+	}
+	updateGerrit(cfg, review, changeId, patchset)
+}
+
+func main() {
+	// Logging happens in syslog because it's almost impossible to get
+	// output out of Gerrit hooks otherwise.
+	log, err := syslog.New(syslog.LOG_INFO|syslog.LOG_USER, "besadii")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to open syslog: %s\n", err)
+		os.Exit(1)
+	}
+
+	log.Info(fmt.Sprintf("besadii called with arguments: %v", os.Args))
+
+	bin := path.Base(os.Args[0])
+	cfg, err := loadConfig()
+
+	if err != nil {
+		log.Crit(fmt.Sprintf("besadii configuration error: %v", err))
+		os.Exit(4)
+	}
+
+	if bin == "patchset-created" {
+		trigger, err := buildTriggerFromPatchsetCreated(cfg)
+		if err != nil {
+			log.Crit(fmt.Sprintf("failed to parse 'patchset-created' invocation from args: %v", err))
+			os.Exit(1)
+		}
+		gerritHookMain(cfg, log, trigger)
+	} else if bin == "change-merged" {
+		trigger, err := buildTriggerFromChangeMerged(cfg)
+		if err != nil {
+			log.Crit(fmt.Sprintf("failed to parse 'change-merged' invocation from args: %v", err))
+			os.Exit(1)
+		}
+		gerritHookMain(cfg, log, trigger)
+	} else if bin == "post-command" {
+		postCommandMain(cfg)
+	} else {
+		fmt.Fprintf(os.Stderr, "besadii does not know how to be invoked as %q, sorry!", bin)
+		os.Exit(1)
+	}
+}