diff options
Diffstat (limited to 'ops/besadii/main.go')
| -rw-r--r-- | ops/besadii/main.go | 459 |
1 files changed, 333 insertions, 126 deletions
diff --git a/ops/besadii/main.go b/ops/besadii/main.go index f862757504..809acc29e8 100644 --- a/ops/besadii/main.go +++ b/ops/besadii/main.go @@ -8,7 +8,7 @@ // // Gerrit (ref-updated) hook: // - Trigger Buildkite CI builds -// - Trigger SourceGraph (cs.tvl.fyi) repository index updates +// - Trigger SourceGraph repository index updates // // Buildkite (post-command) hook: // - Submit CL verification status back to Gerrit @@ -19,31 +19,61 @@ import ( "encoding/json" "flag" "fmt" - "io/ioutil" + "io" "log/syslog" "net/http" + "net/mail" "os" + "os/user" "path" "regexp" + "strconv" + "strings" ) -var branchRegexp = regexp.MustCompile(`^refs/heads/(.*)$`) -var metaRegexp = regexp.MustCompile(`^refs/changes/\d{0,2}/(\d+)/meta$`) -var patchsetRegexp = regexp.MustCompile(`^refs/changes/\d{0,2}/(\d+)/(\d+)$`) +// Regular expression to extract change ID out of a URL +var changeIdRegexp = regexp.MustCompile(`^.*/(\d+)$`) + +// Regular expression to check if gerritChangeName valid. The +// limitation could be what is allowed for a git branch name. For now +// we want to have a stricter limitation for readability and ease of +// use. +var gerritChangeNameRegexp = `^[a-z0-9]+$` +var gerritChangeNameCheck = regexp.MustCompile(gerritChangeNameRegexp) + +// besadii configuration file structure +type config struct { + // Required configuration for Buildkite<>Gerrit monorepo + // integration. + Repository string `json:"repository"` + Branch string `json:"branch"` + GerritUrl string `json:"gerritUrl"` + GerritUser string `json:"gerritUser"` + GerritPassword string `json:"gerritPassword"` + GerritLabel string `json:"gerritLabel"` + BuildkiteOrg string `json:"buildkiteOrg"` + BuildkiteProject string `json:"buildkiteProject"` + BuildkiteToken string `json:"buildkiteToken"` + GerritChangeName string `json:"gerritChangeName"` + + // Optional configuration for Sourcegraph trigger updates. + SourcegraphUrl string `json:"sourcegraphUrl"` + SourcegraphToken string `json:"sourcegraphToken"` +} -// refUpdated is a struct representing the information passed to -// besadii when it is invoked as Gerrit's refUpdated hook. +// buildTrigger represents the information passed to besadii when it +// is invoked as a Gerrit hook. // -// https://gerrit.googlesource.com/plugins/hooks/+/HEAD/src/main/resources/Documentation/hooks.md#ref_updated -type refUpdated struct { - project string - ref string - commit string - submitter string - email string - - changeId *string - patchset *string +// https://gerrit.googlesource.com/plugins/hooks/+/HEAD/src/main/resources/Documentation/hooks.md +type buildTrigger struct { + project string + ref string + commit string + author string + email string + + changeId string + patchset string } type Author struct { @@ -77,65 +107,149 @@ type reviewInput struct { OmitDuplicateComments bool `json:"omit_duplicate_comments"` IgnoreDefaultAttentionSetRules bool `json:"ignore_default_attention_set_rules"` Tag string `json:"tag"` + Notify string `json:"notify,omitempty"` +} + +func defaultConfigLocation() (string, error) { + usr, err := user.Current() + if err != nil { + return "", fmt.Errorf("failed to get current user: %w", err) + } + + return path.Join(usr.HomeDir, "besadii.json"), nil +} + +func loadConfig() (*config, error) { + configPath := os.Getenv("BESADII_CONFIG") + + if configPath == "" { + var err error + configPath, err = defaultConfigLocation() + if err != nil { + return nil, fmt.Errorf("failed to get config location: %w", err) + } + } + + configJson, err := os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("failed to load besadii config: %w", err) + } + + var cfg config + err = json.Unmarshal(configJson, &cfg) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal besadii config: %w", err) + } + + // The default Gerrit label to set is 'Verified', unless specified otherwise. + if cfg.GerritLabel == "" { + cfg.GerritLabel = "Verified" + } + + // The default text referring to a Gerrit Change in BuildKite. + if cfg.GerritChangeName == "" { + cfg.GerritChangeName = "cl" + } + if !gerritChangeNameCheck.MatchString(cfg.GerritChangeName) { + return nil, fmt.Errorf("invalid 'gerritChangeName': %s", cfg.GerritChangeName) + } + + // Rudimentary config validation logic + if cfg.SourcegraphUrl != "" && cfg.SourcegraphToken == "" { + return nil, fmt.Errorf("'SourcegraphToken' must be set if 'SourcegraphUrl' is set") + } + + if cfg.Repository == "" || cfg.Branch == "" { + return nil, fmt.Errorf("missing repository configuration (required: repository, branch)") + } + + if cfg.GerritUrl == "" || cfg.GerritUser == "" || cfg.GerritPassword == "" { + return nil, fmt.Errorf("missing Gerrit configuration (required: gerritUrl, gerritUser, gerritPassword)") + } + + if cfg.BuildkiteOrg == "" || cfg.BuildkiteProject == "" || cfg.BuildkiteToken == "" { + return nil, fmt.Errorf("mising Buildkite configuration (required: buildkiteOrg, buildkiteProject, buildkiteToken)") + } + + return &cfg, nil +} + +// linkToChange creates the full link to a change's patchset in Gerrit +func linkToChange(cfg *config, changeId, patchset string) string { + return fmt.Sprintf("%s/c/%s/+/%s/%s", cfg.GerritUrl, cfg.Repository, changeId, patchset) } // updateGerrit posts a comment on a Gerrit CL to indicate the current build status. -func updateGerrit(review reviewInput, changeId, patchset string) { +func updateGerrit(cfg *config, review reviewInput, changeId, patchset string) { body, _ := json.Marshal(review) - reader := ioutil.NopCloser(bytes.NewReader(body)) + reader := io.NopCloser(bytes.NewReader(body)) - url := fmt.Sprintf("https://cl.tvl.fyi/a/changes/%s/revisions/%s/review", changeId, patchset) + url := fmt.Sprintf("%s/a/changes/%s/revisions/%s/review", cfg.GerritUrl, changeId, patchset) req, err := http.NewRequest("POST", url, reader) if err != nil { - fmt.Fprintf(os.Stderr, "failed to create an HTTP request: %w", err) + fmt.Fprintf(os.Stderr, "failed to create an HTTP request: %s", err) os.Exit(1) } - req.SetBasicAuth("buildkite", gerritPassword()) + req.SetBasicAuth(cfg.GerritUser, cfg.GerritPassword) req.Header.Add("Content-Type", "application/json") resp, err := http.DefaultClient.Do(req) if err != nil { - fmt.Errorf("failed to update CL on Gerrit: %w", err) + fmt.Fprintf(os.Stderr, "failed to update %s on %s: %s", cfg.GerritChangeName, cfg.GerritUrl, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - respBody, _ := ioutil.ReadAll(resp.Body) + respBody, _ := io.ReadAll(resp.Body) fmt.Fprintf(os.Stderr, "received non-success response from Gerrit: %s (%v)", respBody, resp.Status) } else { - fmt.Printf("Added CI status comment on https://cl.tvl.fyi/c/depot/+/%s/%s", changeId, patchset) + fmt.Printf("Added CI status comment on %s", linkToChange(cfg, changeId, patchset)) } } // Trigger a build of a given branch & commit on Buildkite -func triggerBuild(log *syslog.Writer, token string, update *refUpdated) error { +func triggerBuild(cfg *config, log *syslog.Writer, trigger *buildTrigger) error { env := make(map[string]string) + branch := trigger.ref - if update.changeId != nil && update.patchset != nil { - env["GERRIT_CHANGE_ID"] = *update.changeId - env["GERRIT_PATCHSET"] = *update.patchset + // Pass information about the originating Gerrit change to the + // build, if it is for a patchset. + // + // This information is later used by besadii when invoked by Gerrit + // to communicate the build status back to Gerrit. + headBuild := true + if trigger.changeId != "" && trigger.patchset != "" { + env["GERRIT_CHANGE_URL"] = linkToChange(cfg, trigger.changeId, trigger.patchset) + env["GERRIT_CHANGE_ID"] = trigger.changeId + env["GERRIT_PATCHSET"] = trigger.patchset + headBuild = false + + // The branch doesn't have to be a real ref (it's just used to + // group builds), so make it the identifier for the CL. + branch = fmt.Sprintf("%s/%v", cfg.GerritChangeName, strings.Split(trigger.ref, "/")[3]) } build := Build{ - Commit: update.commit, - Branch: update.ref, + Commit: trigger.commit, + Branch: branch, Env: env, Author: Author{ - Name: update.submitter, - Email: update.email, + Name: trigger.author, + Email: trigger.email, }, } body, _ := json.Marshal(build) - reader := ioutil.NopCloser(bytes.NewReader(body)) + reader := io.NopCloser(bytes.NewReader(body)) - req, err := http.NewRequest("POST", "https://api.buildkite.com/v2/organizations/tvl/pipelines/depot/builds", reader) + bkUrl := fmt.Sprintf("https://api.buildkite.com/v2/organizations/%s/pipelines/%s/builds", cfg.BuildkiteOrg, cfg.BuildkiteProject) + req, err := http.NewRequest("POST", bkUrl, reader) if err != nil { return fmt.Errorf("failed to create an HTTP request: %w", err) } - req.Header.Add("Authorization", "Bearer "+token) + req.Header.Add("Authorization", "Bearer "+cfg.BuildkiteToken) req.Header.Add("Content-Type", "application/json") resp, err := http.DefaultClient.Do(req) @@ -145,7 +259,7 @@ func triggerBuild(log *syslog.Writer, token string, update *refUpdated) error { } defer resp.Body.Close() - respBody, err := ioutil.ReadAll(resp.Body) + respBody, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("failed to read Buildkite response body: %w", err) } @@ -160,11 +274,16 @@ func triggerBuild(log *syslog.Writer, token string, update *refUpdated) error { return fmt.Errorf("failed to unmarshal build response: %w", err) } - fmt.Fprintf(log, "triggered build for ref %q at commit %q: %s", update.ref, update.commit, buildResp.WebUrl) + fmt.Fprintf(log, "triggered build for ref %q at commit %q: %s", trigger.ref, trigger.commit, buildResp.WebUrl) + + // For builds of the HEAD branch there is nothing else to do + if headBuild { + return nil + } // Report the status back to the Gerrit CL so that users can click // through to the running build. - msg := fmt.Sprintf("Started build for patchset #%s of cl/%s: %s", *update.patchset, *update.changeId, buildResp.WebUrl) + msg := fmt.Sprintf("Started build for patchset #%s on: %s", trigger.patchset, buildResp.WebUrl) review := reviewInput{ Message: msg, OmitDuplicateComments: true, @@ -172,128 +291,183 @@ func triggerBuild(log *syslog.Writer, token string, update *refUpdated) error { // Do not update the attention set for this comment. IgnoreDefaultAttentionSetRules: true, + + Notify: "NONE", } - updateGerrit(review, *update.changeId, *update.patchset) + updateGerrit(cfg, review, trigger.changeId, trigger.patchset) return nil } -// Trigger a Sourcegraph repository index update on cs.tvl.fyi. +// Trigger a Sourcegraph repository index update. // // https://docs.sourcegraph.com/admin/repo/webhooks -func triggerIndexUpdate(token string) error { - req, err := http.NewRequest("POST", "https://cs.tvl.fyi/.api/repos/depot/-/refresh", nil) +func triggerIndexUpdate(cfg *config, log *syslog.Writer) error { + req, err := http.NewRequest("POST", cfg.SourcegraphUrl, nil) if err != nil { return err } - req.Header.Add("Authorization", "token "+token) + req.Header.Add("Authorization", "token "+cfg.SourcegraphToken) _, err = http.DefaultClient.Do(req) - return err + if err != nil { + return fmt.Errorf("failed to trigger Sourcegraph index update: %w", err) + } + + log.Info("triggered sourcegraph index update") + return nil } -func refUpdatedFromFlags() (*refUpdated, error) { - var update refUpdated +// Gerrit passes more flags than we want, but Rob Pike decided[0] in +// 2013 that the Go art project will not allow users to ignore flags +// because he "doesn't like it". This function allows users to ignore +// flags. +// +// [0]: https://github.com/golang/go/issues/6112#issuecomment-66083768 +func ignoreFlags(ignore []string) { + for _, f := range ignore { + flag.String(f, "", "flag to ignore") + } +} - flag.StringVar(&update.project, "project", "", "Gerrit project") - flag.StringVar(&update.commit, "newrev", "", "new revision") - flag.StringVar(&update.email, "submitter", "", "Submitter email") - flag.StringVar(&update.submitter, "submitter-username", "", "Submitter username") - flag.StringVar(&update.ref, "refname", "", "updated reference name") +// Extract the username & email from Gerrit's uploader flag and set it +// on the trigger struct, for display in Buildkite. +func extractChangeUploader(uploader string, trigger *buildTrigger) error { + // Gerrit passes the uploader in another extra layer of quotes. + uploader, err := strconv.Unquote(uploader) + if err != nil { + return fmt.Errorf("failed to unquote email - forgot quotes on manual invocation?: %w", err) + } - // Gerrit passes more flags than we want, but Rob Pike decided[0] in - // 2013 that the Go art project will not allow users to ignore flags - // because he "doesn't like it". The following code ignores the - // flags. - // - // [0]: https://github.com/golang/go/issues/6112#issuecomment-66083768 - var _old string - flag.StringVar(&_old, "oldrev", "", "") + // Extract the uploader username & email from the input passed by + // Gerrit (in RFC 5322 format). + addr, err := mail.ParseAddress(uploader) + if err != nil { + return fmt.Errorf("invalid change uploader (%s): %w", uploader, err) + } - flag.Parse() + trigger.author = addr.Name + trigger.email = addr.Address - if update.project == "" || update.ref == "" || update.commit == "" || update.submitter == "" { - // If we get here, the user is probably being a dummy and invoking - // this manually - but incorrectly. - return nil, fmt.Errorf("'ref-updated' hook invoked without required arguments") - } + return nil +} - if update.project != "depot" || metaRegexp.MatchString(update.ref) { - // this is not an error, but also not something we handle. +// Extract the buildtrigger struct out of the flags passed to besadii +// when invoked as Gerrit's 'patchset-created' hook. This hook is used +// for triggering CI on in-progress CLs. +func buildTriggerFromPatchsetCreated(cfg *config) (*buildTrigger, error) { + // Information that needs to be returned + var trigger buildTrigger + + // Information that is only needed for parsing + var targetBranch, changeUrl, uploader, kind string + + flag.StringVar(&trigger.project, "project", "", "Gerrit project") + flag.StringVar(&trigger.commit, "commit", "", "commit hash") + flag.StringVar(&trigger.patchset, "patchset", "", "patchset ID") + + flag.StringVar(&targetBranch, "branch", "", "CL target branch") + flag.StringVar(&changeUrl, "change-url", "", "HTTPS URL of change") + flag.StringVar(&uploader, "uploader", "", "Change uploader name & email") + flag.StringVar(&kind, "kind", "", "Kind of patchset") + + // patchset-created also passes various flags which we don't need. + ignoreFlags([]string{"topic", "change", "uploader-username", "change-owner", "change-owner-username"}) + + flag.Parse() + + // Ignore patchsets which do not contain code changes + if kind == "NO_CODE_CHANGE" || kind == "NO_CHANGE" { return nil, nil } - if branchRegexp.MatchString(update.ref) { - // these refs don't need special handling, just move on - return &update, nil + // Parse username & email + err := extractChangeUploader(uploader, &trigger) + if err != nil { + return nil, err } - if matches := patchsetRegexp.FindStringSubmatch(update.ref); matches != nil { - update.changeId = &matches[1] - update.patchset = &matches[2] - return &update, nil + // If the patchset is not for the HEAD branch of the monorepo, then + // we can ignore it. It might be some other kind of change + // (refs/meta/config or Gerrit-internal), but it is not an error. + if trigger.project != cfg.Repository || targetBranch != cfg.Branch { + return nil, nil } - return nil, fmt.Errorf("besadii does not support updates for this type of ref (%q)", update.ref) + // Change ID is not directly passed in the numeric format, so we + // need to extract it out of the URL + matches := changeIdRegexp.FindStringSubmatch(changeUrl) + trigger.changeId = matches[1] + + // Construct the CL ref from which the build should happen. + changeId, _ := strconv.Atoi(trigger.changeId) + trigger.ref = fmt.Sprintf( + "refs/changes/%02d/%s/%s", + changeId%100, trigger.changeId, trigger.patchset, + ) + + return &trigger, nil } -func refUpdatedMain() { - // Logging happens in syslog for Gerrit hooks because we don't want - // the hook output to be intermingled with Gerrit's own output - // stream - log, err := syslog.New(syslog.LOG_INFO|syslog.LOG_USER, "besadii") - if err != nil { - fmt.Fprintf(os.Stderr, "failed to open syslog: %s\n", err) - os.Exit(1) - } +// Extract the buildtrigger struct out of the flags passed to besadii +// when invoked as Gerrit's 'change-merged' hook. This hook is used +// for triggering HEAD builds after change submission. +func buildTriggerFromChangeMerged(cfg *config) (*buildTrigger, error) { + // Information that needs to be returned + var trigger buildTrigger - update, err := refUpdatedFromFlags() - if err != nil { - log.Err(fmt.Sprintf("failed to parse ref update: %s", err)) - os.Exit(1) - } + // Information that is only needed for parsing + var targetBranch, submitter string - if update == nil { // the project was not 'depot' - log.Err("build triggers are only supported for the 'depot' project") - os.Exit(0) - } + flag.StringVar(&trigger.project, "project", "", "Gerrit project") + flag.StringVar(&trigger.commit, "commit", "", "Commit hash") + flag.StringVar(&submitter, "submitter", "", "Submitter email & username") + flag.StringVar(&targetBranch, "branch", "", "CL target branch") - buildkiteToken, err := ioutil.ReadFile("/etc/secrets/buildkite-besadii") - if err != nil { - log.Alert(fmt.Sprintf("buildkite token could not be read: %s", err)) - os.Exit(1) - } + // Ignore extra flags passed by change-merged + ignoreFlags([]string{"change", "topic", "change-url", "submitter-username", "newrev", "change-owner", "change-owner-username"}) - sourcegraphToken, err := ioutil.ReadFile("/etc/secrets/sourcegraph-token") - if err != nil { - log.Alert(fmt.Sprintf("sourcegraph token could not be read: %s", err)) - os.Exit(1) - } + flag.Parse() - err = triggerBuild(log, string(buildkiteToken), update) + // Parse username & email + err := extractChangeUploader(submitter, &trigger) if err != nil { - log.Err(fmt.Sprintf("failed to trigger Buildkite build: %s", err)) + return nil, err } - err = triggerIndexUpdate(string(sourcegraphToken)) - if err != nil { - log.Err(fmt.Sprintf("failed to trigger sourcegraph index update: %s", err)) + // If the patchset is not for the HEAD branch of the monorepo, then + // we can ignore it. + if trigger.project != cfg.Repository || targetBranch != cfg.Branch { + return nil, nil } - log.Info("triggered sourcegraph index update") + + trigger.ref = "refs/heads/" + targetBranch + + return &trigger, nil } -func gerritPassword() string { - gerritPassword, err := ioutil.ReadFile("/etc/secrets/buildkite-gerrit") +func gerritHookMain(cfg *config, log *syslog.Writer, trigger *buildTrigger) { + if trigger == nil { + // The hook was not for something we care about. + os.Exit(0) + } + + err := triggerBuild(cfg, log, trigger) + if err != nil { - fmt.Fprintf(os.Stderr, "Gerrit password could not be read: %s", err) - os.Exit(1) + log.Err(fmt.Sprintf("failed to trigger Buildkite build: %s", err)) } - return string(gerritPassword) + if cfg.SourcegraphUrl != "" && trigger.ref == cfg.Branch { + err = triggerIndexUpdate(cfg, log) + if err != nil { + log.Err(fmt.Sprintf("failed to trigger sourcegraph index update: %s", err)) + } + } } -func postCommandMain() { +func postCommandMain(cfg *config) { changeId := os.Getenv("GERRIT_CHANGE_ID") patchset := os.Getenv("GERRIT_PATCHSET") @@ -301,7 +475,7 @@ func postCommandMain() { // If these variables are unset, but the hook was invoked, the // build was most likely for a branch and not for a CL - no status // needs to be reported back to Gerrit! - fmt.Println("This isn't a CL build, nothing to do. Have a nice day!") + fmt.Printf("This isn't a %s build, nothing to do. Have a nice day!\n", cfg.GerritChangeName) return } @@ -310,15 +484,18 @@ func postCommandMain() { return } - var verified int + var vote int var verb string + var notify string if os.Getenv("BUILDKITE_COMMAND_EXIT_STATUS") == "0" { - verified = 1 // Verified: +1 in Gerrit + vote = 1 // automation passed: +1 in Gerrit verb = "passed" + notify = "NONE" } else { - verified = -1 + vote = -1 verb = "failed" + notify = "OWNER" } msg := fmt.Sprintf("Build of patchset %s %s: %s", patchset, verb, os.Getenv("BUILDKITE_BUILD_URL")) @@ -326,24 +503,54 @@ func postCommandMain() { Message: msg, OmitDuplicateComments: true, Labels: map[string]int{ - "Verified": verified, + cfg.GerritLabel: vote, }, // Update the attention set if we are failing this patchset. - IgnoreDefaultAttentionSetRules: verified == 1, + IgnoreDefaultAttentionSetRules: vote == 1, Tag: "autogenerated:buildkite~result", + + Notify: notify, } - updateGerrit(review, changeId, patchset) + updateGerrit(cfg, review, changeId, patchset) } func main() { + // Logging happens in syslog because it's almost impossible to get + // output out of Gerrit hooks otherwise. + log, err := syslog.New(syslog.LOG_INFO|syslog.LOG_USER, "besadii") + if err != nil { + fmt.Fprintf(os.Stderr, "failed to open syslog: %s\n", err) + os.Exit(1) + } + + log.Info(fmt.Sprintf("besadii called with arguments: %v", os.Args)) + bin := path.Base(os.Args[0]) + cfg, err := loadConfig() + + if err != nil { + log.Crit(fmt.Sprintf("besadii configuration error: %v", err)) + os.Exit(4) + } - if bin == "ref-updated" { - refUpdatedMain() + if bin == "patchset-created" { + trigger, err := buildTriggerFromPatchsetCreated(cfg) + if err != nil { + log.Crit(fmt.Sprintf("failed to parse 'patchset-created' invocation from args: %v", err)) + os.Exit(1) + } + gerritHookMain(cfg, log, trigger) + } else if bin == "change-merged" { + trigger, err := buildTriggerFromChangeMerged(cfg) + if err != nil { + log.Crit(fmt.Sprintf("failed to parse 'change-merged' invocation from args: %v", err)) + os.Exit(1) + } + gerritHookMain(cfg, log, trigger) } else if bin == "post-command" { - postCommandMain() + postCommandMain(cfg) } else { fmt.Fprintf(os.Stderr, "besadii does not know how to be invoked as %q, sorry!", bin) os.Exit(1) |