feat(tools): add rust-crates-advisory r/2595

We have a bunch of crates in `third_party/rust-crates`; it would be
great if we could check them for existing CVEs.

This tool does that, it takes the rust security advisory database,
parses the applicable CVEs, and cross-checks them against the actual
crate versions we list in our package database.

The dumb parser we wrote is tested against all entries in the
database, so we will notice when upstream breaks their shit.
Checking the semver stuff is easy enough with the semver crate.

If an advisory matches, it prints the whole thing and fails the build.

Change-Id: I9e912c43d37a685d9d7a4424defc467a171ea3c4
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2818
Tested-by: BuildkiteCI
Reviewed-by: tazjin <mail@tazj.in>
Reviewed-by: sterni <sternenseemann@systemli.org>

8 files changed, 208 insertions, 11 deletions

diff --git a/third_party/rust-crates/default.nix b/third_party/rust-crates/default.nix
index 3d96c54bd5..66db9331c5 100644
--- a/third_party/rust-crates/default.nix
+++ b/third_party/rust-crates/default.nix
@@ -283,4 +283,31 @@ with depot.third_party.rust-crates;
     sha256 = "1zgl8l15i19lzp90icgwyi6zqdd31b9vm8w129f41d1zd0hs7ayq";
     dependencies = [ log serde ];
   };
+
+  semver-parser = buildRustCrate {
+    pname = "semver-parser";
+    version = "0.7.0";
+    crateName = "semver-parser";
+    edition = "2015";
+    sha256 = "1da66c8413yakx0y15k8c055yna5lyb6fr0fw9318kdwkrk5k12h";
+    dependencies = [ ];
+  };
+
+  semver = buildRustCrate {
+    pname = "semver";
+    version = "0.10.0";
+    crateName = "semver";
+    edition = "2015";
+    sha256 = "0pbkdwlpq4d0hgdrymm2rcw31plni2siwd882gbcbscjvyvrrrqa";
+    dependencies = [ semver-parser ];
+  };
+
+  toml = buildRustCrate {
+    pname = "toml";
+    version = "0.5.8";
+    crateName = "toml";
+    sha256 = "1vwjwmwsy83pbgvvm11a6grbhb09zkcrv9v95wfwv48wjm01wdj4";
+    edition = "2018";
+    dependencies = [ serde ];
+  };
 }
diff --git a/tools/eprintf.nix b/tools/eprintf.nix
new file mode 100644
index 0000000000..eeacca4c8c
--- /dev/null
+++ b/tools/eprintf.nix
@@ -0,0 +1,9 @@
+{ depot, pkgs, ... }:
+
+let
+  bins = depot.nix.getBins pkgs.coreutils [ "printf" ];
+
+# printf(1), but redirect to stderr
+in depot.nix.writeExecline "eprintf" {} [
+  "fdmove" "-c" "1" "2" bins.printf "$@"
+]
diff --git a/tools/rust-crates-advisory/OWNERS b/tools/rust-crates-advisory/OWNERS
new file mode 100644
index 0000000000..a742d0d22b
--- /dev/null
+++ b/tools/rust-crates-advisory/OWNERS
@@ -0,0 +1,3 @@
+inherited: true
+owners:
+  - Profpatsch
diff --git a/tools/rust-crates-advisory/check-security-advisory.rs b/tools/rust-crates-advisory/check-security-advisory.rs
new file mode 100644
index 0000000000..3fd9bc2dd9
--- /dev/null
+++ b/tools/rust-crates-advisory/check-security-advisory.rs
@@ -0,0 +1,67 @@
+extern crate semver;
+extern crate toml;
+
+use std::io::Write;
+
+/// reads a security advisory of the form
+/// https://github.com/RustSec/advisory-db/blob/a24932e220dfa9be8b0b501210fef8a0bc7ef43e/EXAMPLE_ADVISORY.md
+/// and a crate version number,
+/// and returns 0 if the crate version is patched
+/// and returns 1 if the crate version is *not* patched
+///
+/// If PRINT_ADVISORY is set, the advisory is printed if it matches.
+
+fn main() {
+    let mut args = std::env::args_os();
+    let file = args.nth(1).expect("security advisory md file is $1");
+    let crate_version =
+        args.nth(0).expect("crate version is $2")
+        .into_string().expect("crate version string not utf8")
+        ;
+    let crate_version = semver::Version::parse(&crate_version).expect(&format!("this is not a semver version: {}", &crate_version));
+    let filename = file.to_string_lossy();
+
+    let content = std::fs::read(&file).expect(&format!("could not read {}", filename));
+    let content =
+        std::str::from_utf8(&content).expect(&format!("file {} was not encoded as utf-8", filename));
+    let content = content.trim_start();
+
+    let toml_start = content
+        .strip_prefix("```toml").expect(&format!("file did not start with ```toml: {}", filename));
+    let toml_end_index = toml_start.find("```").expect(&format!("the toml section did not end, no `` found: {}", filename));
+    let toml = &toml_start[..toml_end_index];
+    let toml : toml::Value = toml::de::from_slice(toml.as_bytes()).expect(&format!("could not parse toml: {}", filename));
+
+    let versions = toml
+        .as_table().expect(&format!("the toml is not a table: {}", filename))
+        .get("versions").expect(&format!("the toml does not contain the versions field: {}", filename))
+        .as_table().expect(&format!("the toml versions field must be a table: {}", filename));
+
+    let unaffected = match versions.get("unaffected") {
+        Some(u) => u
+            .as_array().expect(&format!("the toml versions.unaffected field must be a list of semvers: {}", filename))
+            .iter()
+            .map(|v| semver::VersionReq::parse(v.as_str().expect(&format!("the version field {} is not a string", v))).expect(&format!("the version field {} is not a valid semver VersionReq", v)))
+            .collect(),
+        None => vec![]
+    };
+
+    let mut patched : Vec<semver::VersionReq> = versions.get("patched").expect(&format!("the toml versions.patched field must exist: {}", filename))
+        .as_array().expect(&format!("the toml versions.patched field must be a list of semvers: {}", filename))
+        .iter()
+        .map(|v| semver::VersionReq::parse(v.as_str().expect(&format!("the version field {} is not a string", v))).expect(&format!("the version field {} is not a valid semver VersionReq", v)))
+        .collect();
+
+    patched.extend_from_slice(&unaffected[..]);
+    let is_patched_or_unaffected = patched.iter().any(|req| req.matches(&crate_version));
+
+    if is_patched_or_unaffected {
+        std::process::exit(0);
+    } else {
+        if std::env::var_os("PRINT_ADVISORY").is_some() {
+            write!(std::io::stderr(), "Advisory {} matched!\n{}\n", filename, content).unwrap();
+        }
+        std::process::exit(1);
+    }
+
+}
diff --git a/tools/rust-crates-advisory/default.nix b/tools/rust-crates-advisory/default.nix
new file mode 100644
index 0000000000..d9b8783904
--- /dev/null
+++ b/tools/rust-crates-advisory/default.nix
@@ -0,0 +1,97 @@
+{ depot, pkgs, lib, ... }:
+
+let
+
+  bins =
+       depot.nix.getBins pkgs.s6-portable-utils [ "s6-ln" "s6-cat" "s6-echo" "s6-mkdir" "s6-test" "s6-touch" ]
+    // depot.nix.getBins pkgs.lr [ "lr" ]
+    ;
+
+  crate-advisories = "${pkgs.fetchFromGitHub {
+    owner = "RustSec";
+    repo = "advisory-db";
+    # TODO(Profpatsch): this will have to be updated regularly, how?
+    rev = "113188c62380753f01ff0df5edb7d67a300b143a";
+    sha256 = "0v086ybwr71zgs5nv8yr4w2w2d4daxx6in2s1sjb4m41q1r9p0wj";
+  }}/crates";
+
+  our-crates = lib.mapAttrsToList (_: lib.id)
+    # this is a bit eh, but no idea how to avoid the readTree thing otherwise
+    (builtins.removeAttrs depot.third_party.rust-crates [ "__readTree" ]);
+
+  check-security-advisory = depot.nix.writers.rustSimple {
+    name = "parse-security-advisory";
+    dependencies = [
+      depot.third_party.rust-crates.toml
+      depot.third_party.rust-crates.semver
+    ];
+  } (builtins.readFile ./check-security-advisory.rs);
+
+  # $1 is the directory with advisories for crate $2 with version $3
+  check-crate-advisory = depot.nix.writeExecline "check-crate-advisory" { readNArgs = 3; } [
+    "pipeline" [ bins.lr "-0" "-t" "depth == 1" "$1" ]
+    "forstdin" "-0" "-Eo" "0" "advisory"
+    "if" [ depot.tools.eprintf "advisory %s\n" "$advisory" ]
+    check-security-advisory "$advisory" "$3"
+  ];
+
+  # Run through everything in the `crate-advisories` repository
+  # and check whether we can parse all the advisories without crashing.
+  test-parsing-all-security-advisories = depot.nix.runExecline "check-all-our-crates" {} [
+    "pipeline" [ bins.lr "-0" "-t" "depth == 1" crate-advisories ]
+    "if" [
+      # this will succeed as long as check-crate-advisory doesn’t `panic!()` (status 101)
+      "forstdin" "-0" "-E" "-x" "101" "crate_advisories"
+      check-crate-advisory "$crate_advisories" "foo" "0.0.0"
+    ]
+    "importas" "out" "out"
+    bins.s6-touch "$out"
+  ];
+
+
+  check-all-our-crates = depot.nix.runExecline "check-all-our-crates" {
+    stdin = lib.concatStrings
+      (map
+        (crate:
+          depot.nix.netstring.fromString
+            ( depot.nix.netstring.fromString crate.crateName
+            + depot.nix.netstring.fromString crate.version ))
+        our-crates);
+  } [
+    "if" [
+      "forstdin" "-o" "0" "-Ed" "" "crateNetstring"
+      "multidefine" "-d" "" "$crateNetstring" [ "crate" "crate_version" ]
+      "if" [ depot.tools.eprintf "checking %s, version %s\n" "$crate" "$crate_version" ]
+
+      "ifthenelse" [ bins.s6-test "-d" "${crate-advisories}/\${crate}" ]
+          [ # also print the full advisory text if it matches
+            "export" "PRINT_ADVISORY" "1"
+            check-crate-advisory "${crate-advisories}/\${crate}" "$crate" "$crate_version"
+          ]
+        [ depot.tools.eprintf "No advisories found for crate %s\n" "$crate" ]
+        "importas" "-ui" "ret" "?"
+        # put a marker in ./failed to read at the end
+        "ifelse" [ bins.s6-test "$ret" "-eq" "1" ]
+          [ bins.s6-touch "./failed" ]
+        "if" [ depot.tools.eprintf "\n" ]
+        "exit" "$ret"
+    ]
+    "ifelse" [ bins.s6-test "-f" "./failed" ]
+      [ "if" [ depot.tools.eprintf "Error: Found active advisories!" ]
+        "exit" "1"
+      ]
+    "importas" "out" "out"
+    bins.s6-touch "$out"
+  ];
+
+in depot.nix.utils.drvTargets {
+
+  check-all-our-crates =
+    depot.nix.drvSeqL
+      [ test-parsing-all-security-advisories ]
+      check-all-our-crates;
+
+  inherit
+    check-crate-advisory
+    ;
+}
diff --git a/users/Profpatsch/blog/default.nix b/users/Profpatsch/blog/default.nix
index 584c12d8d7..9d22e7f770 100644
--- a/users/Profpatsch/blog/default.nix
+++ b/users/Profpatsch/blog/default.nix
@@ -81,7 +81,7 @@ let
     me.netencode.record-splice-env
     runOr return500
     "importas" "-i" "path" "path"
-    "if" [ me.lib.eprintf "GET \${path}\n" ]
+    "if" [ depot.tools.eprintf "GET \${path}\n" ]
     runOr return404
     "backtick" "-ni" "TEMPLATE_DATA" [
       "ifelse" [ bins.test "$path" "=" "/notes" ]
@@ -118,7 +118,7 @@ let
     "importas" "?" "?"
     "ifelse" [ bins.test "$?" "-eq" "0" ]
     []
-    "if" [ me.lib.eprintf "runOr: exited \${?}, running \${1}\n" ]
+    "if" [ depot.tools.eprintf "runOr: exited \${?}, running \${1}\n" ]
     "$1"
   ];
 
diff --git a/users/Profpatsch/lib.nix b/users/Profpatsch/lib.nix
index 1e9652a279..5d5fb01294 100644
--- a/users/Profpatsch/lib.nix
+++ b/users/Profpatsch/lib.nix
@@ -13,10 +13,6 @@ let
     "$@"
   ];
 
-  eprintf = depot.nix.writeExecline "eprintf" {} [
-    "fdmove" "-c" "1" "2" bins.printf "$@"
-  ];
-
   eprint-stdin = depot.nix.writeExecline "eprint-stdin" {} [
     "pipeline" [ bins.multitee "0-1,2" ] "$@"
   ];
@@ -37,7 +33,7 @@ let
   eprintenv = depot.nix.writeExecline "eprintenv" { readNArgs = 1; } [
     "ifelse" [ "fdmove" "-c" "1" "2" bins.printenv "$1" ]
     [ "$@" ]
-    "if" [ eprintf "eprintenv: could not find \"\${1}\" in the environment\n" ]
+    "if" [ depot.tools.eprintf "eprintenv: could not find \"\${1}\" in the environment\n" ]
     "$@"
   ];
 
@@ -54,7 +50,6 @@ let
 in {
   inherit
     debugExec
-    eprintf
     eprint-stdin
     eprint-stdin-netencode
     eprintenv
diff --git a/users/Profpatsch/nixpkgs-rewriter/default.nix b/users/Profpatsch/nixpkgs-rewriter/default.nix
index ff414862fa..9dac018441 100644
--- a/users/Profpatsch/nixpkgs-rewriter/default.nix
+++ b/users/Profpatsch/nixpkgs-rewriter/default.nix
@@ -5,7 +5,6 @@ let
     ;
   inherit (depot.users.Profpatsch.lib)
     debugExec
-    eprintf
     ;
 
   bins = depot.nix.getBins pkgs.coreutils [ "head" "shuf" ]
@@ -41,7 +40,7 @@ let
     "importas" "-ui" "file" "fileName"
     "importas" "-ui" "from" "fromLine"
     "importas" "-ui" "to" "toLine"
-    "if" [ eprintf "%s-%s\n" "$from" "$to" ]
+    "if" [ depot.tools.eprintf "%s-%s\n" "$from" "$to" ]
     (debugExec "adding lib")
     bins.sed
       "-e" "\${from},\${to} \${1}"
@@ -98,7 +97,7 @@ let
     "pipeline" [ bins.shuf ]
     "pipeline" [ bins.head "-n" "1000" ]
     bins.xargs "-I" "{}" "-n1"
-    "if" [ eprintf "instantiating %s\n" "{}" ]
+    "if" [ depot.tools.eprintf "instantiating %s\n" "{}" ]
     "nix-instantiate" "$1" "-A" "{}"
   ];