feat(web/bubblegum): nix CGI programming framework r/2394

So here is what has been keeping me up at night: At some point I
realized that nix actually made a somewhat passable language for CGI
programming:

* That `builtins.getEnv` exists as one of the impurities of Nix is
  perfect as environment variables are the main way of communication
  from the web server to the CGI application.

* We can actually read from the filesystem via builtins.readDir and
  builtins.readFile with bearable overhead if we avoid importing the
  used paths into the nix store.

* Templating and routing are convenient to implement via indented strings
  and attribute sets respectively.

Of course there are obvious limitation:

* The overhead of derivations is probably much to great for them to be
  useful via IfD.

* Even without derivations, nix evaluation is very slow to the point
  were a trivial application takes between 100ms and 400ms to produce a
  response.

* We can't really cause effects other than producing a response which
  makes it not viable for a lot of applications. There are some ways
  around this:

  * With a custom interpreter we could have streaming and multiplexed
    I/O (using lazy lists emulated via attrsets) to cause such effects,
    but it would probably perform terribly.

  * We can use builtins.fetchurl to call other HTTP-based microservices,
    but only in very limited constraints, i. e. only GET, no headers,
    and only if the tarball ttl is set to 0 in the global nix.conf.

* Terrible error handling capabilities because builtins.tryEval actually
  doesn't catch a lot of errors.

To prove that it actually works, there are some demo applications,
which I invite you to run and potentially break horribly:

    nix-build -A web.bubblegum.examples && ./result
    # navigate to http://localhost:9000

The setup uses thttpd and executes the nix CGI scripts using
users.sterni.nint which automatically passed `depot`, so they can
import the cgi library.

Change-Id: I3a22a749612211627e5f8301c31ec2e7a872812c
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2746
Tested-by: BuildkiteCI
Reviewed-by: tazjin <mail@tazj.in>

1 files changed, 221 insertions, 0 deletions

diff --git a/web/bubblegum/default.nix b/web/bubblegum/default.nix
new file mode 100644
index 000000000000..81e41cfbdfbd
--- /dev/null
+++ b/web/bubblegum/default.nix
@@ -0,0 +1,221 @@
+{ depot, lib, pkgs, ... }:
+
+let
+
+  inherit (depot.nix.yants)
+    defun
+    restrict
+    struct
+    string
+    int
+    attrs
+    enum
+    ;
+
+  inherit (depot.nix)
+    runExecline
+    getBins
+    ;
+
+  headers = attrs string;
+
+  statusCodes = {
+    # 1xx
+    "Continue" = 100;
+    "Switching Protocols" = 101;
+    "Processing" = 102;
+    "Early Hints" = 103;
+    # 2xx
+    "OK" = 200;
+    "Created" = 201;
+    "Accepted" = 202;
+    "Non-Authoritative Information" = 203;
+    "No Content" = 204;
+    "Reset Content" = 205;
+    "Partial Content" = 206;
+    "Multi Status" = 207;
+    "Already Reported" = 208;
+    "IM Used" = 226;
+    # 3xx
+    "Multiple Choices" = 300;
+    "Moved Permanently" = 301;
+    "Found" = 302;
+    "See Other" = 303;
+    "Not Modified" = 304;
+    "Use Proxy" = 305;
+    "Switch Proxy" = 306;
+    "Temporary Redirect" = 307;
+    "Permanent Redirect" = 308;
+    # 4xx
+    "Bad Request" = 400;
+    "Unauthorized" = 401;
+    "Payment Required" = 402;
+    "Forbidden" = 403;
+    "Not Found" = 404;
+    "Method Not Allowed" = 405;
+    "Not Acceptable" = 406;
+    "Proxy Authentication Required" = 407;
+    "Request Timeout" = 408;
+    "Conflict" = 409;
+    "Gone" = 410;
+    "Length Required" = 411;
+    "Precondition Failed" = 412;
+    "Payload Too Large" = 413;
+    "URI Too Long" = 414;
+    "Unsupported Media Type" = 415;
+    "Range Not Satisfiable" = 416;
+    "Expectation Failed" = 417;
+    "I'm a teapot" = 418;
+    "Misdirected Request" = 421;
+    "Unprocessable Entity" = 422;
+    "Locked" = 423;
+    "Failed Dependency" = 424;
+    "Too Early" = 425;
+    "Upgrade Required" = 426;
+    "Precondition Required" = 428;
+    "Too Many Requests" = 429;
+    "Request Header Fields Too Large" = 431;
+    "Unavailable For Legal Reasons" = 451;
+    # 5xx
+    "Internal Server Error" = 500;
+    "Not Implemented" = 501;
+    "Bad Gateway" = 502;
+    "Service Unavailable" = 503;
+    "Gateway Timeout" = 504;
+    "HTTP Version Not Supported" = 505;
+    "Variant Also Negotiates" = 506;
+    "Insufficient Storage" = 507;
+    "Loop Detected" = 508;
+    "Not Extended" = 510;
+    "Network Authentication Required" = 511;
+  };
+
+  status = enum "bubblegum.status"
+    (builtins.attrNames statusCodes);
+
+  /* Generate a CGI response. Takes three arguments:
+
+     1. Status of the response as a string which is
+        the descriptive name in the protocol, e. g.
+        `"OK"`, `"Not Found"` etc.
+     2. Attribute set describing extra headers to
+        send, keys and values should both be strings.
+     3. Response content as a string.
+
+     See the [README](./README.md) for an example.
+
+    Type: Status -> Headers -> Body -> string
+  */
+  respond = defun [ status headers string string ]
+    (s: hs: body:
+      let
+        code = status.match s statusCodes;
+        renderedHeaders = lib.concatStrings
+          (lib.mapAttrsToList (n: v: "${n}: ${v}\r\n") hs);
+      in
+        lib.concatStrings [
+          "Status: ${toString code} ${s}\r\n"
+          renderedHeaders
+          "\r\n"
+          body
+        ]);
+
+  /* Returns the value of the `SCRIPT_NAME` environment
+     variable used by CGI.
+  */
+  scriptName = builtins.getEnv "SCRIPT_NAME";
+
+  /* Returns the value of the `PATH_INFO` environment
+     variable used by CGI. All cases that could be
+     considered as the CGI script's root (i. e.
+     `PATH_INFO` is empty or `/`) is mapped to `"/"`
+     for convenience.
+  */
+  pathInfo =
+    let
+      p = builtins.getEnv "PATH_INFO";
+    in
+      if builtins.stringLength p == 0
+      then "/"
+      else p;
+
+  /* Helper function which converts a path from the
+     root of the CGI script (i. e. something which
+     could be the content of `PATH_INFO`) to an
+     absolute path from the web root by also
+     utilizing `scriptName`.
+
+     Type: string -> string
+  */
+  absolutePath = defun [ string string ]
+    (path:
+      if builtins.substring 0 1 path == "/"
+      then "${scriptName}${path}"
+      else "${scriptName}/${path}");
+
+  bins = getBins pkgs.coreutils [ "env" "tee" "cat" "printf" "chmod" ]
+      // getBins depot.users.sterni.nint [ "nint" ];
+
+  /* Type: args -> either path derivation string -> derivation
+  */
+  writeCGI =
+    { # if given sets the `PATH` to search for `nix-instantiate`
+      # Useful when using for example thttpd which unsets `PATH`
+      # in the CGI environment.
+      binPath ? ""
+      # name of the resulting derivation. Defaults to `baseNameOf`
+      # the input path or name of the input derivation.
+      # Must be given if the input is a string.
+    , name ? null
+    }:
+    input: let
+      drvName =
+        if name != null
+        then name
+        else if builtins.isPath input
+        then baseNameOf input
+        else if lib.isDerivation input
+        then input.name
+        else builtins.throw "Need name";
+      script =
+        if builtins.isPath input || lib.isDerivation input
+        then input
+        else if builtins.isString input
+        then pkgs.writeText "${drvName}-source" input
+        else builtins.throw "Unsupported input: ${lib.generators.toPretty {} input}";
+      shebang = lib.concatStringsSep " " ([
+        "#!${bins.env}"
+        # use the slightly cursed /usr/bin/env -S which allows us
+        # to pass any number of arguments to our interpreter
+        # instead of maximum one using plain shebang which considers
+        # everything after the first space as the second argument.
+        "-S"
+      ] ++ lib.optionals (builtins.stringLength binPath > 0) [
+        "PATH=${binPath}"
+      ] ++ [
+        "${bins.nint}"
+        # always pass depot so scripts can use this library
+        "--arg depot '(import ${depot.depotPath} {})'"
+      ]);
+    in runExecline.local drvName {} [
+      "importas" "out" "out"
+      "pipeline" [
+        "foreground" [
+          "if" [ bins.printf "%s\n" shebang ]
+        ]
+        "if" [ bins.cat script ]
+      ]
+      "if" [ bins.tee "$out" ]
+      "if" [ bins.chmod "+x" "$out" ]
+      "exit" "0"
+    ];
+
+in {
+  inherit
+    respond
+    pathInfo
+    scriptName
+    absolutePath
+    writeCGI
+    ;
+}