about summary refs log tree commit diff
path: root/nix/buildkite
diff options
context:
space:
mode:
authorVincent Ambo <mail@tazj.in>2022-01-30T16·06+0300
committertazjin <tazjin@tvl.su>2022-01-31T16·11+0000
commitaa122cbae78ce97d60c0c98ba14df753d97e40b1 (patch)
tree12b98d85c4b18fe870feb26de70db9ba61837bd7 /nix/buildkite
parent2d10d60fac0fd00a71b65cfdcb9fba0477b2086c (diff)
style: format entire depot with nixpkgs-fmt r/3723
This CL can be used to compare the style of nixpkgs-fmt against other
formatters (nixpkgs, alejandra).

Change-Id: I87c6abff6bcb546b02ead15ad0405f81e01b6d9e
Reviewed-on: https://cl.tvl.fyi/c/depot/+/4397
Tested-by: BuildkiteCI
Reviewed-by: sterni <sternenseemann@systemli.org>
Reviewed-by: lukegb <lukegb@tvl.fyi>
Reviewed-by: wpcarro <wpcarro@gmail.com>
Reviewed-by: Profpatsch <mail@profpatsch.de>
Reviewed-by: kanepyork <rikingcoding@gmail.com>
Reviewed-by: tazjin <tazjin@tvl.su>
Reviewed-by: cynthia <cynthia@tvl.fyi>
Reviewed-by: edef <edef@edef.eu>
Reviewed-by: eta <tvl@eta.st>
Reviewed-by: grfn <grfn@gws.fyi>
Diffstat (limited to 'nix/buildkite')
-rw-r--r--nix/buildkite/default.nix312
1 files changed, 164 insertions, 148 deletions
diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix
index e0c947deae..d17b5c86c4 100644
--- a/nix/buildkite/default.nix
+++ b/nix/buildkite/default.nix
@@ -29,7 +29,8 @@ let
     unsafeDiscardStringContext;
 
   inherit (pkgs) lib runCommandNoCC writeText;
-in rec {
+in
+rec {
   # Creates a Nix expression that yields the target at the specified
   # location in the repository.
   #
@@ -42,14 +43,15 @@ in rec {
       descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})";
       targetExpr = foldl' descend "import ./. {}" target.__readTree;
       subtargetExpr = descend targetExpr target.__subtarget;
-    in if target ? __subtarget then subtargetExpr else targetExpr;
+    in
+    if target ? __subtarget then subtargetExpr else targetExpr;
 
   # Create a pipeline label from the target's tree location.
   mkLabel = target:
     let label = concatStringsSep "/" target.__readTree;
     in if target ? __subtarget
-      then "${label}:${target.__subtarget}"
-      else label;
+    then "${label}:${target.__subtarget}"
+    else label;
 
   # Determine whether to skip a target if it has not diverged from the
   # HEAD branch.
@@ -74,33 +76,36 @@ in rec {
 
   # Create a pipeline step from a single target.
   mkStep = headBranch: parentTargetMap: target:
-  let
-    label = mkLabel target;
-    drvPath = unsafeDiscardStringContext target.drvPath;
-    shouldSkip' = shouldSkip parentTargetMap;
-  in {
-    label = ":nix: " + label;
-    key = hashString "sha1" label;
-    skip = shouldSkip' label drvPath;
-    command = mkBuildCommand target drvPath;
-    env.READTREE_TARGET = label;
-
-    # Add a dependency on the initial static pipeline step which
-    # always runs. This allows build steps uploaded in batches to
-    # start running before all batches have been uploaded.
-    depends_on = ":init:";
-  };
+    let
+      label = mkLabel target;
+      drvPath = unsafeDiscardStringContext target.drvPath;
+      shouldSkip' = shouldSkip parentTargetMap;
+    in
+    {
+      label = ":nix: " + label;
+      key = hashString "sha1" label;
+      skip = shouldSkip' label drvPath;
+      command = mkBuildCommand target drvPath;
+      env.READTREE_TARGET = label;
+
+      # Add a dependency on the initial static pipeline step which
+      # always runs. This allows build steps uploaded in batches to
+      # start running before all batches have been uploaded.
+      depends_on = ":init:";
+    };
 
   # Helper function to inelegantly divide a list into chunks of at
   # most n elements.
   #
   # This works by assigning each element a chunk ID based on its
   # index, and then grouping all elements by their chunk ID.
-  chunksOf = n: list: let
-    chunkId = idx: toString (idx / n + 1);
-    assigned = lib.imap1 (idx: value: { inherit value ; chunk = chunkId idx; }) list;
-    unchunk = mapAttrs (_: elements: map (e: e.value) elements);
-  in unchunk (lib.groupBy (e: e.chunk) assigned);
+  chunksOf = n: list:
+    let
+      chunkId = idx: toString (idx / n + 1);
+      assigned = lib.imap1 (idx: value: { inherit value; chunk = chunkId idx; }) list;
+      unchunk = mapAttrs (_: elements: map (e: e.value) elements);
+    in
+    unchunk (lib.groupBy (e: e.chunk) assigned);
 
   # Define a build pipeline chunk as a JSON file, using the pipeline
   # format documented on
@@ -120,104 +125,112 @@ in rec {
     attrValues (mapAttrs (makePipelineChunk name) (chunksOf 192 steps));
 
   # Create a pipeline structure for the given targets.
-  mkPipeline = {
-    # HEAD branch of the repository on which release steps, GC
-    # anchoring and other "mainline only" steps should run.
-    headBranch,
-
-    # List of derivations as read by readTree (in most cases just the
-    # output of readTree.gather) that should be built in Buildkite.
-    #
-    # These are scheduled as the first build steps and run as fast as
-    # possible, in order, without any concurrency restrictions.
-    drvTargets,
-
-    # Derivation map of a parent commit. Only targets which no longer
-    # correspond to the content of this map will be built. Passing an
-    # empty map will always build all targets.
-    parentTargetMap ? {},
-
-    # A list of plain Buildkite step structures to run alongside the
-    # build for all drvTargets, but before proceeding with any
-    # post-build actions such as status reporting.
-    #
-    # Can be used for things like code formatting checks.
-    additionalSteps ? [],
-
-    # A list of plain Buildkite step structures to run after all
-    # previous steps succeeded.
-    #
-    # Can be used for status reporting steps and the like.
-    postBuildSteps ? []
-  }: let
-    # Convert a target into all of its build and post-build steps,
-    # treated separately as they need to be in different chunks.
-    targetToSteps = target: let
-      step = mkStep headBranch parentTargetMap target;
-
-      # Split build/post-build steps
-      splitExtraSteps = partition ({ postStep, ... }: postStep)
-       (attrValues (mapAttrs (name: value: {
-         inherit name value;
-         postStep = (value ? prompt) || (value.postBuild or false);
-        }) (target.meta.ci.extraSteps or {})));
-
-      mkExtraStep' = { name, value, ... }: mkExtraStep step name value;
-      extraBuildSteps = map mkExtraStep' splitExtraSteps.wrong; # 'wrong' -> no prompt
-      extraPostSteps = map mkExtraStep' splitExtraSteps.right; # 'right' -> has prompt
-    in {
-      buildSteps = [ step ] ++ extraBuildSteps;
-      postSteps = extraPostSteps;
-    };
-
-    # Combine all target steps into separate build and post-build step lists.
-    steps = foldl' (acc: t: {
-      buildSteps = acc.buildSteps ++ t.buildSteps;
-      postSteps = acc.postSteps ++ t.postSteps;
-    }) { buildSteps = []; postSteps = []; } (map targetToSteps drvTargets);
-
-    buildSteps =
-      # Add build steps for each derivation target and their extra
-      # steps.
-      steps.buildSteps
-
-      # Add additional steps (if set).
-      ++ additionalSteps;
-
-    postSteps =
-      # Add post-build steps for each derivation target.
-      steps.postSteps
-
-      # Add any globally defined post-build steps.
-      ++ postBuildSteps;
-
-    buildChunks = pipelineChunks "build" buildSteps;
-    postBuildChunks = pipelineChunks "post" postSteps;
-    chunks = buildChunks ++ postBuildChunks;
-  in runCommandNoCC "buildkite-pipeline" {} ''
-    mkdir $out
-    echo "Generated ${toString (length chunks)} pipeline chunks"
-    ${
-      lib.concatMapStringsSep "\n"
-        (chunk: "cp ${chunk.path} $out/${chunk.filename}") chunks
-    }
-  '';
+  mkPipeline =
+    {
+      # HEAD branch of the repository on which release steps, GC
+      # anchoring and other "mainline only" steps should run.
+      headBranch
+    , # List of derivations as read by readTree (in most cases just the
+      # output of readTree.gather) that should be built in Buildkite.
+      #
+      # These are scheduled as the first build steps and run as fast as
+      # possible, in order, without any concurrency restrictions.
+      drvTargets
+    , # Derivation map of a parent commit. Only targets which no longer
+      # correspond to the content of this map will be built. Passing an
+      # empty map will always build all targets.
+      parentTargetMap ? { }
+    , # A list of plain Buildkite step structures to run alongside the
+      # build for all drvTargets, but before proceeding with any
+      # post-build actions such as status reporting.
+      #
+      # Can be used for things like code formatting checks.
+      additionalSteps ? [ ]
+    , # A list of plain Buildkite step structures to run after all
+      # previous steps succeeded.
+      #
+      # Can be used for status reporting steps and the like.
+      postBuildSteps ? [ ]
+    }:
+    let
+      # Convert a target into all of its build and post-build steps,
+      # treated separately as they need to be in different chunks.
+      targetToSteps = target:
+        let
+          step = mkStep headBranch parentTargetMap target;
+
+          # Split build/post-build steps
+          splitExtraSteps = partition ({ postStep, ... }: postStep)
+            (attrValues (mapAttrs
+              (name: value: {
+                inherit name value;
+                postStep = (value ? prompt) || (value.postBuild or false);
+              })
+              (target.meta.ci.extraSteps or { })));
+
+          mkExtraStep' = { name, value, ... }: mkExtraStep step name value;
+          extraBuildSteps = map mkExtraStep' splitExtraSteps.wrong; # 'wrong' -> no prompt
+          extraPostSteps = map mkExtraStep' splitExtraSteps.right; # 'right' -> has prompt
+        in
+        {
+          buildSteps = [ step ] ++ extraBuildSteps;
+          postSteps = extraPostSteps;
+        };
+
+      # Combine all target steps into separate build and post-build step lists.
+      steps = foldl'
+        (acc: t: {
+          buildSteps = acc.buildSteps ++ t.buildSteps;
+          postSteps = acc.postSteps ++ t.postSteps;
+        })
+        { buildSteps = [ ]; postSteps = [ ]; }
+        (map targetToSteps drvTargets);
+
+      buildSteps =
+        # Add build steps for each derivation target and their extra
+        # steps.
+        steps.buildSteps
+
+        # Add additional steps (if set).
+        ++ additionalSteps;
+
+      postSteps =
+        # Add post-build steps for each derivation target.
+        steps.postSteps
+
+        # Add any globally defined post-build steps.
+        ++ postBuildSteps;
+
+      buildChunks = pipelineChunks "build" buildSteps;
+      postBuildChunks = pipelineChunks "post" postSteps;
+      chunks = buildChunks ++ postBuildChunks;
+    in
+    runCommandNoCC "buildkite-pipeline" { } ''
+      mkdir $out
+      echo "Generated ${toString (length chunks)} pipeline chunks"
+      ${
+        lib.concatMapStringsSep "\n"
+          (chunk: "cp ${chunk.path} $out/${chunk.filename}") chunks
+      }
+    '';
 
   # Create a drvmap structure for the given targets, containing the
   # mapping of all target paths to their derivations. The mapping can
   # be persisted for future use.
-  mkDrvmap = drvTargets: writeText "drvmap.json" (toJSON (listToAttrs (map (target: {
-    name = mkLabel target;
-    value = {
-      drvPath = unsafeDiscardStringContext target.drvPath;
-
-      # Include the attrPath in the output to reconstruct the drv
-      # without parsing the human-readable label.
-      attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [
-        target.__subtarget
-      ];
-    };
-  }) drvTargets)));
+  mkDrvmap = drvTargets: writeText "drvmap.json" (toJSON (listToAttrs (map
+    (target: {
+      name = mkLabel target;
+      value = {
+        drvPath = unsafeDiscardStringContext target.drvPath;
+
+        # Include the attrPath in the output to reconstruct the drv
+        # without parsing the human-readable label.
+        attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [
+          target.__subtarget
+        ];
+      };
+    })
+    drvTargets)));
 
   # Implementation of extra step logic.
   #
@@ -278,34 +291,37 @@ in rec {
 
   # Create the Buildkite configuration for an extra step, optionally
   # wrapping it in a gate group.
-  mkExtraStep = parent: key: {
-    command,
-    label ? key,
-    prompt ? false,
-    needsOutput ? false,
-    branches ? null,
-    alwaysRun ? false,
-    postBuild ? false
-  }@cfg: let
-    parentLabel = parent.env.READTREE_TARGET;
-
-    step = {
-      label = ":gear: ${label} (from ${parentLabel})";
-      skip = if alwaysRun then false else parent.skip or false;
-      depends_on = lib.optional (!alwaysRun && !needsOutput) parent.key;
-      branches = if branches != null then lib.concatStringsSep " " branches else null;
-
-      command = pkgs.writeShellScript "${key}-script" ''
-        set -ueo pipefail
-        ${lib.optionalString needsOutput "echo '~~~ Preparing build output of ${parentLabel}'"}
-        ${lib.optionalString needsOutput parent.command}
-        echo '+++ Running extra step command'
-        exec ${command}
-      '';
-    };
-  in if (isString prompt)
-    then mkGatedStep {
-      inherit step label parent prompt;
-    }
+  mkExtraStep = parent: key: { command
+                             , label ? key
+                             , prompt ? false
+                             , needsOutput ? false
+                             , branches ? null
+                             , alwaysRun ? false
+                             , postBuild ? false
+                             }@cfg:
+    let
+      parentLabel = parent.env.READTREE_TARGET;
+
+      step = {
+        label = ":gear: ${label} (from ${parentLabel})";
+        skip = if alwaysRun then false else parent.skip or false;
+        depends_on = lib.optional (!alwaysRun && !needsOutput) parent.key;
+        branches = if branches != null then lib.concatStringsSep " " branches else null;
+
+        command = pkgs.writeShellScript "${key}-script" ''
+          set -ueo pipefail
+          ${lib.optionalString needsOutput "echo '~~~ Preparing build output of ${parentLabel}'"}
+          ${lib.optionalString needsOutput parent.command}
+          echo '+++ Running extra step command'
+          exec ${command}
+        '';
+      };
+    in
+    if (isString prompt)
+    then
+      mkGatedStep
+        {
+          inherit step label parent prompt;
+        }
     else step;
 }