// Copyright 2019-2020 Google LLC.
// SPDX-License-Identifier: Apache-2.0
//
// besadii is a small CLI tool that is invoked as a hook by various
// programs to cause CI-related actions.
//
// It supports the following modes & operations:
//
// Gerrit (ref-updated) hook:
// - Trigger Buildkite CI builds
// - Trigger SourceGraph repository index updates
//
// Buildkite (post-command) hook:
// - Submit CL verification status back to Gerrit
package main
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log/syslog"
"net/http"
"net/mail"
"os"
"os/user"
"path"
"regexp"
"strconv"
"strings"
)
// Regular expression to extract change ID out of a URL
var changeIdRegexp = regexp.MustCompile(`^.*/(\d+)$`)
// Regular expression to check if gerritChangeName valid. The
// limitation could be what is allowed for a git branch name. For now
// we want to have a stricter limitation for readability and ease of
// use.
var gerritChangeNameRegexp = `^[a-z0-9]+$`
var gerritChangeNameCheck = regexp.MustCompile(gerritChangeNameRegexp)
// besadii configuration file structure
type config struct {
// Required configuration for Buildkite<>Gerrit monorepo
// integration.
Repository string `json:"repository"`
Branch string `json:"branch"`
GerritUrl string `json:"gerritUrl"`
GerritUser string `json:"gerritUser"`
GerritPassword string `json:"gerritPassword"`
GerritLabel string `json:"gerritLabel"`
BuildkiteOrg string `json:"buildkiteOrg"`
BuildkiteProject string `json:"buildkiteProject"`
BuildkiteToken string `json:"buildkiteToken"`
GerritChangeName string `json:"gerritChangeName"`
// Optional configuration for Sourcegraph trigger updates.
SourcegraphUrl string `json:"sourcegraphUrl"`
SourcegraphToken string `json:"sourcegraphToken"`
}
// buildTrigger represents the information passed to besadii when it
// is invoked as a Gerrit hook.
//
// https://gerrit.googlesource.com/plugins/hooks/+/HEAD/src/main/resources/Documentation/hooks.md
type buildTrigger struct {
project string
ref string
commit string
author string
email string
changeId string
patchset string
}
type Author struct {
Name string `json:"name"`
Email string `json:"email"`
}
// Build is the representation of a Buildkite build as described on
// https://buildkite.com/docs/apis/rest-api/builds#create-a-build
type Build struct {
Commit string `json:"commit"`
Branch string `json:"branch"`
Author Author `json:"author"`
Env map[string]string `json:"env"`
}
// BuildResponse is the representation of Buildkite's success response
// after triggering a build. This has many fields, but we only need
// one of them.
type buildResponse struct {
WebUrl string `json:"web_url"`
}
// reviewInput is a struct representing the data submitted to Gerrit
// to post a review on a CL.
//
// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input
type reviewInput struct {
Message string `json:"message"`
Labels map[string]int `json:"labels,omitempty"`
OmitDuplicateComments bool `json:"omit_duplicate_comments"`
IgnoreDefaultAttentionSetRules bool `json:"ignore_default_attention_set_rules"`
Tag string `json:"tag"`
Notify string `json:"notify,omitempty"`
}
func defaultConfigLocation() (string, error) {
usr, err := user.Current()
if err != nil {
return "", fmt.Errorf("failed to get current user: %w", err)
}
return path.Join(usr.HomeDir, "besadii.json"), nil
}
func loadConfig() (*config, error) {
configPath := os.Getenv("BESADII_CONFIG")
if configPath == "" {
var err error
configPath, err = defaultConfigLocation()
if err != nil {
return nil, fmt.Errorf("failed to get config location: %w", err)
}
}
configJson, err := ioutil.ReadFile(configPath)
if err != nil {
return nil, fmt.Errorf("failed to load besadii config: %w", err)
}
var cfg config
err = json.Unmarshal(configJson, &cfg)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal besadii config: %w", err)
}
// The default Gerrit label to set is 'Verified', unless specified otherwise.
if cfg.GerritLabel == "" {
cfg.GerritLabel = "Verified"
}
// The default text referring to a Gerrit Change in BuildKite.
if cfg.GerritChangeName == "" {
cfg.GerritChangeName = "cl"
}
if !gerritChangeNameCheck.MatchString(cfg.GerritChangeName) {
return nil, fmt.Errorf("invalid 'gerritChangeName': %s", cfg.GerritChangeName)
}
// Rudimentary config validation logic
if cfg.SourcegraphUrl != "" && cfg.SourcegraphToken == "" {
return nil, fmt.Errorf("'SourcegraphToken' must be set if 'SourcegraphUrl' is set")
}
if cfg.Repository == "" || cfg.Branch == "" {
return nil, fmt.Errorf("missing repository configuration (required: repository, branch)")
}
if cfg.GerritUrl == "" || cfg.GerritUser == "" || cfg.GerritPassword == "" {
return nil, fmt.Errorf("missing Gerrit configuration (required: gerritUrl, gerritUser, gerritPassword)")
}
if cfg.BuildkiteOrg == "" || cfg.BuildkiteProject == "" || cfg.BuildkiteToken == "" {
return nil, fmt.Errorf("mising Buildkite configuration (required: buildkiteOrg, buildkiteProject, buildkiteToken)")
}
return &cfg, nil
}
// linkToChange creates the full link to a change's patchset in Gerrit
func linkToChange(cfg *config, changeId, patchset string) string {
return fmt.Sprintf("%s/c/%s/+/%s/%s", cfg.GerritUrl, cfg.Repository, changeId, patchset)
}
// updateGerrit posts a comment on a Gerrit CL to indicate the current build status.
func updateGerrit(cfg *config, review reviewInput, changeId, patchset string) {
body, _ := json.Marshal(review)
reader := ioutil.NopCloser(bytes.NewReader(body))
url := fmt.Sprintf("%s/a/changes/%s/revisions/%s/review", cfg.GerritUrl, changeId, patchset)
req, err := http.NewRequest("POST", url, reader)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to create an HTTP request: %w", err)
os.Exit(1)
}
req.SetBasicAuth(cfg.GerritUser, cfg.GerritPassword)
req.Header.Add("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
fmt.Errorf("failed to update %s on %s: %w", cfg.GerritChangeName, cfg.GerritUrl, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
respBody, _ := ioutil.ReadAll(resp.Body)
fmt.Fprintf(os.Stderr, "received non-success response from Gerrit: %s (%v)", respBody, resp.Status)
} else {
fmt.Printf("Added CI status comment on %s", linkToChange(cfg, changeId, patchset))
}
}
// Trigger a build of a given branch & commit on Buildkite
func triggerBuild(cfg *config, log *syslog.Writer, trigger *buildTrigger) error {
env := make(map[string]string)
branch := trigger.ref
// Pass information about the originating Gerrit change to the
// build, if it is for a patchset.
//
// This information is later used by besadii when invoked by Gerrit
// to communicate the build status back to Gerrit.
headBuild := true
if trigger.changeId != "" && trigger.patchset != "" {
env["GERRIT_CHANGE_URL"] = linkToChange(cfg, trigger.changeId, trigger.patchset)
env["GERRIT_CHANGE_ID"] = trigger.changeId
env["GERRIT_PATCHSET"] = trigger.patchset
headBuild = false
// The branch doesn't have to be a real ref (it's just used to
// group builds), so make it the identifier for the CL.
branch = fmt.Sprintf("%s/%v", cfg.GerritChangeName, strings.Split(trigger.ref, "/")[3])
}
build := Build{
Commit: trigger.commit,
Branch: branch,
Env: env,
Author: Author{
Name: trigger.author,
Email: trigger.email,
},
}
body, _ := json.Marshal(build)
reader := ioutil.NopCloser(bytes.NewReader(body))
bkUrl := fmt.Sprintf("https://api.buildkite.com/v2/organizations/%s/pipelines/%s/builds", cfg.BuildkiteOrg, cfg.BuildkiteProject)
req, err := http.NewRequest("POST", bkUrl, reader)
if err != nil {
return fmt.Errorf("failed to create an HTTP request: %w", err)
}
req.Header.Add("Authorization", "Bearer "+cfg.BuildkiteToken)
req.Header.Add("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
// This might indicate a temporary error on the Buildkite side.
return fmt.Errorf("failed to send Buildkite request: %w", err)
}
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("failed to read Buildkite response body: %w", err)
}
if resp.StatusCode != http.StatusCreated {
return fmt.Errorf("received non-success response from Buildkite: %s (%v)", respBody, resp.Status)
}
var buildResp buildResponse
err = json.Unmarshal(respBody, &buildResp)
if err != nil {
return fmt.Errorf("failed to unmarshal build response: %w", err)
}
fmt.Fprintf(log, "triggered build for ref %q at commit %q: %s", trigger.ref, trigger.commit, buildResp.WebUrl)
// For builds of the HEAD branch there is nothing else to do
if headBuild {
return nil
}
// Report the status back to the Gerrit CL so that users can click
// through to the running build.
msg := fmt.Sprintf("Started build for patchset #%s on: %s", trigger.patchset, buildResp.WebUrl)
review := reviewInput{
Message: msg,
OmitDuplicateComments: true,
Tag: "autogenerated:buildkite~trigger",
// Do not update the attention set for this comment.
IgnoreDefaultAttentionSetRules: true,
Notify: "NONE",
}
updateGerrit(cfg, review, trigger.changeId, trigger.patchset)
return nil
}
// Trigger a Sourcegraph repository index update.
//
// https://docs.sourcegraph.com/admin/repo/webhooks
func triggerIndexUpdate(cfg *config, log *syslog.Writer) error {
req, err := http.NewRequest("POST", cfg.SourcegraphUrl, nil)
if err != nil {
return err
}
req.Header.Add("Authorization", "token "+cfg.SourcegraphToken)
_, err = http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("failed to trigger Sourcegraph index update: %w", err)
}
log.Info("triggered sourcegraph index update")
return nil
}
// Gerrit passes more flags than we want, but Rob Pike decided[0] in
// 2013 that the Go art project will not allow users to ignore flags
// because he "doesn't like it". This function allows users to ignore
// flags.
//
// [0]: https://github.com/golang/go/issues/6112#issuecomment-66083768
func ignoreFlags(ignore []string) {
for _, f := range ignore {
flag.String(f, "", "flag to ignore")
}
}
// Extract the username & email from Gerrit's uploader flag and set it
// on the trigger struct, for display in Buildkite.
func extractChangeUploader(uploader string, trigger *buildTrigger) error {
// Gerrit passes the uploader in another extra layer of quotes.
uploader, err := strconv.Unquote(uploader)
if err != nil {
return fmt.Errorf("failed to unquote email - forgot quotes on manual invocation?: %w", err)
}
// Extract the uploader username & email from the input passed by
// Gerrit (in RFC 5322 format).
addr, err := mail.ParseAddress(uploader)
if err != nil {
return fmt.Errorf("invalid change uploader (%s): %w", uploader, err)
}
trigger.author = addr.Name
trigger.email = addr.Address
return nil
}
// Extract the buildtrigger struct out of the flags passed to besadii
// when invoked as Gerrit's 'patchset-created' hook. This hook is used
// for triggering CI on in-progress CLs.
func buildTriggerFromPatchsetCreated(cfg *config) (*buildTrigger, error) {
// Information that needs to be returned
var trigger buildTrigger
// Information that is only needed for parsing
var targetBranch, changeUrl, uploader, kind string
flag.StringVar(&trigger.project, "project", "", "Gerrit project")
flag.StringVar(&trigger.commit, "commit", "", "commit hash")
flag.StringVar(&trigger.patchset, "patchset", "", "patchset ID")
flag.StringVar(&targetBranch, "branch", "", "CL target branch")
flag.StringVar(&changeUrl, "change-url", "", "HTTPS URL of change")
flag.StringVar(&uploader, "uploader", "", "Change uploader name & email")
flag.StringVar(&kind, "kind", "", "Kind of patchset")
// patchset-created also passes various flags which we don't need.
ignoreFlags([]string{"topic", "change", "uploader-username", "change-owner", "change-owner-username"})
flag.Parse()
// Ignore patchsets which do not contain code changes
if kind == "NO_CODE_CHANGE" || kind == "NO_CHANGE" {
return nil, nil
}
// Parse username & email
err := extractChangeUploader(uploader, &trigger)
if err != nil {
return nil, err
}
// If the patchset is not for the HEAD branch of the monorepo, then
// we can ignore it. It might be some other kind of change
// (refs/meta/config or Gerrit-internal), but it is not an error.
if trigger.project != cfg.Repository || targetBranch != cfg.Branch {
return nil, nil
}
// Change ID is not directly passed in the numeric format, so we
// need to extract it out of the URL
matches := changeIdRegexp.FindStringSubmatch(changeUrl)
trigger.changeId = matches[1]
// Construct the CL ref from which the build should happen.
changeId, _ := strconv.Atoi(trigger.changeId)
trigger.ref = fmt.Sprintf(
"refs/changes/%02d/%s/%s",
changeId%100, trigger.changeId, trigger.patchset,
)
return &trigger, nil
}
// Extract the buildtrigger struct out of the flags passed to besadii
// when invoked as Gerrit's 'change-merged' hook. This hook is used
// for triggering HEAD builds after change submission.
func buildTriggerFromChangeMerged(cfg *config) (*buildTrigger, error) {
// Information that needs to be returned
var trigger buildTrigger
// Information that is only needed for parsing
var targetBranch, submitter string
flag.StringVar(&trigger.project, "project", "", "Gerrit project")
flag.StringVar(&trigger.commit, "commit", "", "Commit hash")
flag.StringVar(&submitter, "submitter", "", "Submitter email & username")
flag.StringVar(&targetBranch, "branch", "", "CL target branch")
// Ignore extra flags passed by change-merged
ignoreFlags([]string{"change", "topic", "change-url", "submitter-username", "newrev", "change-owner", "change-owner-username"})
flag.Parse()
// Parse username & email
err := extractChangeUploader(submitter, &trigger)
if err != nil {
return nil, err
}
// If the patchset is not for the HEAD branch of the monorepo, then
// we can ignore it.
if trigger.project != cfg.Repository || targetBranch != cfg.Branch {
return nil, nil
}
trigger.ref = "refs/heads/" + targetBranch
return &trigger, nil
}
func gerritHookMain(cfg *config, log *syslog.Writer, trigger *buildTrigger) {
if trigger == nil {
// The hook was not for something we care about.
os.Exit(0)
}
err := triggerBuild(cfg, log, trigger)
if err != nil {
log.Err(fmt.Sprintf("failed to trigger Buildkite build: %s", err))
}
if cfg.SourcegraphUrl != "" && trigger.ref == "refs/heads/canon" {
err = triggerIndexUpdate(cfg, log)
if err != nil {
log.Err(fmt.Sprintf("failed to trigger sourcegraph index update: %s", err))
}
}
}
func postCommandMain(cfg *config) {
changeId := os.Getenv("GERRIT_CHANGE_ID")
patchset := os.Getenv("GERRIT_PATCHSET")
if changeId == "" || patchset == "" {
// If these variables are unset, but the hook was invoked, the
// build was most likely for a branch and not for a CL - no status
// needs to be reported back to Gerrit!
fmt.Println("This isn't a %s build, nothing to do. Have a nice day!", cfg.GerritChangeName)
return
}
if os.Getenv("BUILDKITE_LABEL") != ":duck:" {
// this is not the build stage, don't do anything.
return
}
var vote int
var verb string
var notify string
if os.Getenv("BUILDKITE_COMMAND_EXIT_STATUS") == "0" {
vote = 1 // automation passed: +1 in Gerrit
verb = "passed"
notify = "NONE"
} else {
vote = -1
verb = "failed"
notify = "OWNER"
}
msg := fmt.Sprintf("Build of patchset %s %s: %s", patchset, verb, os.Getenv("BUILDKITE_BUILD_URL"))
review := reviewInput{
Message: msg,
OmitDuplicateComments: true,
Labels: map[string]int{
cfg.GerritLabel: vote,
},
// Update the attention set if we are failing this patchset.
IgnoreDefaultAttentionSetRules: vote == 1,
Tag: "autogenerated:buildkite~result",
Notify: notify,
}
updateGerrit(cfg, review, changeId, patchset)
}
func main() {
// Logging happens in syslog because it's almost impossible to get
// output out of Gerrit hooks otherwise.
log, err := syslog.New(syslog.LOG_INFO|syslog.LOG_USER, "besadii")
if err != nil {
fmt.Fprintf(os.Stderr, "failed to open syslog: %s\n", err)
os.Exit(1)
}
log.Info(fmt.Sprintf("besadii called with arguments: %v", os.Args))
bin := path.Base(os.Args[0])
cfg, err := loadConfig()
if err != nil {
log.Crit(fmt.Sprintf("besadii configuration error: %v", err))
os.Exit(4)
}
if bin == "patchset-created" {
trigger, err := buildTriggerFromPatchsetCreated(cfg)
if err != nil {
log.Crit(fmt.Sprintf("failed to parse 'patchset-created' invocation from args: %v", err))
os.Exit(1)
}
gerritHookMain(cfg, log, trigger)
} else if bin == "change-merged" {
trigger, err := buildTriggerFromChangeMerged(cfg)
if err != nil {
log.Crit(fmt.Sprintf("failed to parse 'change-merged' invocation from args: %v", err))
os.Exit(1)
}
gerritHookMain(cfg, log, trigger)
} else if bin == "post-command" {
postCommandMain(cfg)
} else {
fmt.Fprintf(os.Stderr, "besadii does not know how to be invoked as %q, sorry!", bin)
os.Exit(1)
}
}
