diff options
author | Vincent Ambo <mail@tazj.in> | 2021-10-01T10·43+0300 |
---|---|---|
committer | tazjin <tazjin@tvl.su> | 2022-01-17T10·26+0000 |
commit | 0a21da2bb4db308d8cf01f454e7b9c3a01b8947f (patch) | |
tree | 5b672426fbad1b4d0ca03c8449f872e83b54ec50 /nix/buildkite/default.nix | |
parent | 546251678a5206b211c64e5093d2df986f1afc84 (diff) |
feat(ops/pipelines): Create drvmap structure for each commit r/3601
Always create a structure that maps all targets to derivations, and persist it as a JSON file. This relates to some of the ideas expressed in: https://docs.google.com/document/d/16A0a5oUxH1VoiSM8hyFyLW0WiUYpNo2e2D6FTW4BlH8/edit The file is always uploaded to Buildkite as an artifact. This allows for retrieving it based on the commit ID in a Buildkite GraphQL query. By default, Buildkite stores artefacts for 6 months. Storage location can be overridden (with custom retention) through some environment variables, but for now at TVL the Buildkite-managed storage is fine. See also: https://buildkite.com/docs/pipelines/artifacts In the subsequent filtering implementation, when diffing commits across a time-range that exceeds artefact retention time, we should simply default to building everything. Change-Id: I6d808461cd1c1fdd6983ba8c8ef075736d42caa7 Reviewed-on: https://cl.tvl.fyi/c/depot/+/3662 Tested-by: BuildkiteCI Reviewed-by: sterni <sternenseemann@systemli.org>
Diffstat (limited to 'nix/buildkite/default.nix')
-rw-r--r-- | nix/buildkite/default.nix | 26 |
1 files changed, 23 insertions, 3 deletions
diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix index 09fe0a741e41..5a61aef3b779 100644 --- a/nix/buildkite/default.nix +++ b/nix/buildkite/default.nix @@ -15,9 +15,13 @@ let concatStringsSep filter foldl' + getEnv length + listToAttrs mapAttrs - toJSON; + pathExists + toJSON + unsafeDiscardStringContext; inherit (pkgs) lib runCommandNoCC writeText; in rec { @@ -44,7 +48,7 @@ in rec { # Skip build steps if their out path has already been built. skip = headBranch: target: let - shouldSkip = with builtins; + shouldSkip = # Only skip in real Buildkite builds (getEnv "BUILDKITE_BUILD_ID" != "") && # Always build everything for the canon branch. @@ -60,7 +64,7 @@ in rec { skip = if skipIfBuilt then skip headBranch target else false; command = let - drvPath = builtins.unsafeDiscardStringContext target.drvPath; + drvPath = unsafeDiscardStringContext target.drvPath; in concatStringsSep " " [ # First try to realise the drvPath of the target so we don't evaluate twice. # Nix has no concept of depending on a derivation file without depending on @@ -162,4 +166,20 @@ in rec { (chunk: "cp ${chunk.path} $out/${chunk.filename}") chunks } ''; + + # Create a drvmap structure for the given targets, containing the + # mapping of all target paths to their derivations. The mapping can + # be persisted for future use. + mkDrvmap = drvTargets: writeText "drvmap.json" (toJSON (listToAttrs (map (target: { + name = mkLabel target; + value = { + drvPath = unsafeDiscardStringContext target.drvPath; + + # Include the attrPath in the output to reconstruct the drv + # without parsing the human-readable label. + attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [ + target.__subtarget + ]; + }; + }) drvTargets))); } |