From 93a746aaaa092ffc3e7eb37e1df30bfd3a28435f Mon Sep 17 00:00:00 2001 From: sterni Date: Sun, 21 Feb 2021 12:57:40 +0100 Subject: feat(web/bubblegum): nix CGI programming framework So here is what has been keeping me up at night: At some point I realized that nix actually made a somewhat passable language for CGI programming: * That `builtins.getEnv` exists as one of the impurities of Nix is perfect as environment variables are the main way of communication from the web server to the CGI application. * We can actually read from the filesystem via builtins.readDir and builtins.readFile with bearable overhead if we avoid importing the used paths into the nix store. * Templating and routing are convenient to implement via indented strings and attribute sets respectively. Of course there are obvious limitation: * The overhead of derivations is probably much to great for them to be useful via IfD. * Even without derivations, nix evaluation is very slow to the point were a trivial application takes between 100ms and 400ms to produce a response. * We can't really cause effects other than producing a response which makes it not viable for a lot of applications. There are some ways around this: * With a custom interpreter we could have streaming and multiplexed I/O (using lazy lists emulated via attrsets) to cause such effects, but it would probably perform terribly. * We can use builtins.fetchurl to call other HTTP-based microservices, but only in very limited constraints, i. e. only GET, no headers, and only if the tarball ttl is set to 0 in the global nix.conf. * Terrible error handling capabilities because builtins.tryEval actually doesn't catch a lot of errors. To prove that it actually works, there are some demo applications, which I invite you to run and potentially break horribly: nix-build -A web.bubblegum.examples && ./result # navigate to http://localhost:9000 The setup uses thttpd and executes the nix CGI scripts using users.sterni.nint which automatically passed `depot`, so they can import the cgi library. Change-Id: I3a22a749612211627e5f8301c31ec2e7a872812c Reviewed-on: https://cl.tvl.fyi/c/depot/+/2746 Tested-by: BuildkiteCI Reviewed-by: tazjin --- web/bubblegum/README.md | 68 +++++++ web/bubblegum/default.nix | 221 +++++++++++++++++++++ web/bubblegum/examples/blog.nix | 134 +++++++++++++ web/bubblegum/examples/default.nix | 61 ++++++ web/bubblegum/examples/derivation-svg.nix | 11 + web/bubblegum/examples/hello.nix | 80 ++++++++ web/bubblegum/examples/posts/2021-04-01-hello.html | 3 + .../examples/posts/2021-04-02-a second post.html | 7 + 8 files changed, 585 insertions(+) create mode 100644 web/bubblegum/README.md create mode 100644 web/bubblegum/default.nix create mode 100644 web/bubblegum/examples/blog.nix create mode 100644 web/bubblegum/examples/default.nix create mode 100644 web/bubblegum/examples/derivation-svg.nix create mode 100644 web/bubblegum/examples/hello.nix create mode 100644 web/bubblegum/examples/posts/2021-04-01-hello.html create mode 100644 web/bubblegum/examples/posts/2021-04-02-a second post.html (limited to 'web/bubblegum') diff --git a/web/bubblegum/README.md b/web/bubblegum/README.md new file mode 100644 index 0000000000..0e09c1c65e --- /dev/null +++ b/web/bubblegum/README.md @@ -0,0 +1,68 @@ +# //web/bubblegum + +`bubblegum` is a CGI programming library for the Nix expression language. +It provides a few helpers to make writing CGI scripts which are executable +using [//users/sterni/nint](../../users/sterni/nint/README.md) convenient. + +An example nix.cgi script looks like this (don't worry about the shebang +too much, you can use `web.bubblegum.writeCGI` to set this up without +thinking twice): + +```nix +#!/usr/bin/env nint --arg depot '(import /path/to/depot {})' +{ depot, ... }: + +let + inherit (depot.web.bubblegum) + respond + ; +in + +respond "OK" { + "Content-type" = "text/html"; + # further headers… +} '' + + + + + hello world + + + hello world! + + +'' +``` + +As you can see, the core component of `bubblegum` is the `respond` +function which takes three arguments: + +* The response status as the textual representation which is also + returned to the client in the HTTP protocol, e. g. `"OK"`, + `"Not Found"`, `"Bad Request"`, … + +* An attribute set mapping header names to header values to be sent. + +* The response body as a string. + +Additionally it exposes a few helpers for working with the CGI +environment like `pathInfo` which is a wrapper around +`builtins.getEnv "PATH_INFO"`. The documentation for all exposed +helpers is inlined in [default.nix](./default.nix) (you should be +able to use `nixdoc` to render it). + +For deployment purposes it is recommended to use `writeCGI` which +takes a nix CGI script in the form of a derivation, path or string +and builds an executable nix CGI script which has the correct shebang +set and is automatically passed a version of depot from the nix store, +so the script has access to the `bubblegum` library. + +For example nix CGI scripts and a working deployment using `thttpd` +see the [examples directory](./examples). You can also start a local +server running the examples like this: + +``` +$ nix-build -A web.bubblegum.examples && ./result +# navigate to http://localhost:9000 +``` diff --git a/web/bubblegum/default.nix b/web/bubblegum/default.nix new file mode 100644 index 0000000000..81e41cfbdf --- /dev/null +++ b/web/bubblegum/default.nix @@ -0,0 +1,221 @@ +{ depot, lib, pkgs, ... }: + +let + + inherit (depot.nix.yants) + defun + restrict + struct + string + int + attrs + enum + ; + + inherit (depot.nix) + runExecline + getBins + ; + + headers = attrs string; + + statusCodes = { + # 1xx + "Continue" = 100; + "Switching Protocols" = 101; + "Processing" = 102; + "Early Hints" = 103; + # 2xx + "OK" = 200; + "Created" = 201; + "Accepted" = 202; + "Non-Authoritative Information" = 203; + "No Content" = 204; + "Reset Content" = 205; + "Partial Content" = 206; + "Multi Status" = 207; + "Already Reported" = 208; + "IM Used" = 226; + # 3xx + "Multiple Choices" = 300; + "Moved Permanently" = 301; + "Found" = 302; + "See Other" = 303; + "Not Modified" = 304; + "Use Proxy" = 305; + "Switch Proxy" = 306; + "Temporary Redirect" = 307; + "Permanent Redirect" = 308; + # 4xx + "Bad Request" = 400; + "Unauthorized" = 401; + "Payment Required" = 402; + "Forbidden" = 403; + "Not Found" = 404; + "Method Not Allowed" = 405; + "Not Acceptable" = 406; + "Proxy Authentication Required" = 407; + "Request Timeout" = 408; + "Conflict" = 409; + "Gone" = 410; + "Length Required" = 411; + "Precondition Failed" = 412; + "Payload Too Large" = 413; + "URI Too Long" = 414; + "Unsupported Media Type" = 415; + "Range Not Satisfiable" = 416; + "Expectation Failed" = 417; + "I'm a teapot" = 418; + "Misdirected Request" = 421; + "Unprocessable Entity" = 422; + "Locked" = 423; + "Failed Dependency" = 424; + "Too Early" = 425; + "Upgrade Required" = 426; + "Precondition Required" = 428; + "Too Many Requests" = 429; + "Request Header Fields Too Large" = 431; + "Unavailable For Legal Reasons" = 451; + # 5xx + "Internal Server Error" = 500; + "Not Implemented" = 501; + "Bad Gateway" = 502; + "Service Unavailable" = 503; + "Gateway Timeout" = 504; + "HTTP Version Not Supported" = 505; + "Variant Also Negotiates" = 506; + "Insufficient Storage" = 507; + "Loop Detected" = 508; + "Not Extended" = 510; + "Network Authentication Required" = 511; + }; + + status = enum "bubblegum.status" + (builtins.attrNames statusCodes); + + /* Generate a CGI response. Takes three arguments: + + 1. Status of the response as a string which is + the descriptive name in the protocol, e. g. + `"OK"`, `"Not Found"` etc. + 2. Attribute set describing extra headers to + send, keys and values should both be strings. + 3. Response content as a string. + + See the [README](./README.md) for an example. + + Type: Status -> Headers -> Body -> string + */ + respond = defun [ status headers string string ] + (s: hs: body: + let + code = status.match s statusCodes; + renderedHeaders = lib.concatStrings + (lib.mapAttrsToList (n: v: "${n}: ${v}\r\n") hs); + in + lib.concatStrings [ + "Status: ${toString code} ${s}\r\n" + renderedHeaders + "\r\n" + body + ]); + + /* Returns the value of the `SCRIPT_NAME` environment + variable used by CGI. + */ + scriptName = builtins.getEnv "SCRIPT_NAME"; + + /* Returns the value of the `PATH_INFO` environment + variable used by CGI. All cases that could be + considered as the CGI script's root (i. e. + `PATH_INFO` is empty or `/`) is mapped to `"/"` + for convenience. + */ + pathInfo = + let + p = builtins.getEnv "PATH_INFO"; + in + if builtins.stringLength p == 0 + then "/" + else p; + + /* Helper function which converts a path from the + root of the CGI script (i. e. something which + could be the content of `PATH_INFO`) to an + absolute path from the web root by also + utilizing `scriptName`. + + Type: string -> string + */ + absolutePath = defun [ string string ] + (path: + if builtins.substring 0 1 path == "/" + then "${scriptName}${path}" + else "${scriptName}/${path}"); + + bins = getBins pkgs.coreutils [ "env" "tee" "cat" "printf" "chmod" ] + // getBins depot.users.sterni.nint [ "nint" ]; + + /* Type: args -> either path derivation string -> derivation + */ + writeCGI = + { # if given sets the `PATH` to search for `nix-instantiate` + # Useful when using for example thttpd which unsets `PATH` + # in the CGI environment. + binPath ? "" + # name of the resulting derivation. Defaults to `baseNameOf` + # the input path or name of the input derivation. + # Must be given if the input is a string. + , name ? null + }: + input: let + drvName = + if name != null + then name + else if builtins.isPath input + then baseNameOf input + else if lib.isDerivation input + then input.name + else builtins.throw "Need name"; + script = + if builtins.isPath input || lib.isDerivation input + then input + else if builtins.isString input + then pkgs.writeText "${drvName}-source" input + else builtins.throw "Unsupported input: ${lib.generators.toPretty {} input}"; + shebang = lib.concatStringsSep " " ([ + "#!${bins.env}" + # use the slightly cursed /usr/bin/env -S which allows us + # to pass any number of arguments to our interpreter + # instead of maximum one using plain shebang which considers + # everything after the first space as the second argument. + "-S" + ] ++ lib.optionals (builtins.stringLength binPath > 0) [ + "PATH=${binPath}" + ] ++ [ + "${bins.nint}" + # always pass depot so scripts can use this library + "--arg depot '(import ${depot.depotPath} {})'" + ]); + in runExecline.local drvName {} [ + "importas" "out" "out" + "pipeline" [ + "foreground" [ + "if" [ bins.printf "%s\n" shebang ] + ] + "if" [ bins.cat script ] + ] + "if" [ bins.tee "$out" ] + "if" [ bins.chmod "+x" "$out" ] + "exit" "0" + ]; + +in { + inherit + respond + pathInfo + scriptName + absolutePath + writeCGI + ; +} diff --git a/web/bubblegum/examples/blog.nix b/web/bubblegum/examples/blog.nix new file mode 100644 index 0000000000..f79ab0627e --- /dev/null +++ b/web/bubblegum/examples/blog.nix @@ -0,0 +1,134 @@ +{ depot, ... }: + +let + inherit (depot) + lib + ; + + inherit (depot.users.sterni.nix) + url + fun + string + ; + + inherit (depot.web.bubblegum) + pathInfo + scriptName + respond + absolutePath + ; + + # substituted using substituteAll in default.nix + blogdir = "@blogdir@"; + # blogdir = toString ./posts; # for local testing + + parseDate = post: + let + matched = builtins.match "/?([0-9]+)-([0-9]+)-([0-9]+)-.+" post; + in + if matched == null + then [ 0 0 0 ] + else builtins.map builtins.fromJSON matched; + + parseTitle = post: + let + matched = builtins.match "/?[0-9]+-[0-9]+-[0-9]+-(.+).html" post; + in + if matched == null + then "no title" + else builtins.head matched; + + dateAtLeast = a: b: + builtins.all fun.id + (lib.zipListsWith (partA: partB: partA >= partB) a b); + + byPostDate = a: b: + dateAtLeast (parseDate a) (parseDate b); + + posts = builtins.sort byPostDate + (builtins.attrNames + (lib.filterAttrs (_: v: v == "regular") + (builtins.readDir blogdir))); + + generic = { title, inner, ... }: '' + + + + + ${title} + + + + ${inner} + + + ''; + + index = posts: '' +
+

blog posts

+ +
+ ''; + + formatDate = + let + # Assume we never deal with years < 1000 + formatDigit = d: string.fit { + char = "0"; width = 2; + } (toString d); + in lib.concatMapStringsSep "-" formatDigit; + + post = title: post: '' +
+

${title}

+
+ ${builtins.readFile (blogdir + "/" + post)} +
+
+
+

Posted on ${formatDate (parseDate post)}

+ +
+ ''; + + validatePathInfo = pathInfo: + let + chars = string.toChars pathInfo; + in builtins.length chars > 1 + && !(builtins.elem "/" (builtins.tail chars)); + + response = + if pathInfo == "/" + then { + title = "blog"; + status = "OK"; + inner = index posts; + } + else if !(validatePathInfo pathInfo) + then { + title = "Bad Request"; + status = "Bad Request"; + inner = "No slashes in post names 😡"; + } + # CGI should already url.decode for us + else if builtins.pathExists (blogdir + "/" + pathInfo) + then rec { + title = parseTitle pathInfo; + status = "OK"; + inner = post title pathInfo; + } else { + title = "Not Found"; + status = "Not Found"; + inner = "

404 — not found

"; + }; +in + respond response.status { + "Content-type" = "text/html"; + } (generic response) diff --git a/web/bubblegum/examples/default.nix b/web/bubblegum/examples/default.nix new file mode 100644 index 0000000000..3f0f51db63 --- /dev/null +++ b/web/bubblegum/examples/default.nix @@ -0,0 +1,61 @@ +{ depot, pkgs, lib, ... }: + +let + + scripts = [ + ./hello.nix + ./derivation-svg.nix + (substituteAll { + src = ./blog.nix; + # by making this a plain string this + # can be something outside the nix store! + blogdir = ./posts; + }) + ]; + + inherit (depot.nix) + writeExecline + runExecline + getBins + ; + + inherit (depot.web.bubblegum) + writeCGI + ; + + inherit (pkgs) + runCommandLocal + substituteAll + ; + + bins = (getBins pkgs.thttpd [ "thttpd" ]) + // (getBins pkgs.coreutils [ "printf" "cp" "mkdir" ]); + + webRoot = + let + copyScripts = lib.concatMap + (path: let + cgi = writeCGI { + # assume we are on NixOS since thttpd doesn't set PATH. + # using third_party.nix is tricky because not everyone + # has a tvix daemon running. + binPath = "/run/current-system/sw/bin"; + } path; + in [ + "if" [ bins.cp cgi "\${out}/${cgi.name}" ] + ]) scripts; + in runExecline.local "webroot" {} ([ + "importas" "out" "out" + "if" [ bins.mkdir "-p" "$out" ] + ] ++ copyScripts); + + port = 9000; + +in + writeExecline "serve-examples" {} [ + "foreground" [ + bins.printf "%s\n" "Running on http://localhost:${toString port}" + ] + "${bins.thttpd}" "-D" "-p" (toString port) "-l" "/dev/stderr" + "-c" "*.nix" "-d" webRoot + ] diff --git a/web/bubblegum/examples/derivation-svg.nix b/web/bubblegum/examples/derivation-svg.nix new file mode 100644 index 0000000000..a5f30a2bd1 --- /dev/null +++ b/web/bubblegum/examples/derivation-svg.nix @@ -0,0 +1,11 @@ +# Warning: this is *very* slow on the first request +{ depot, ... }: + +let + inherit (depot.web.bubblegum) + respond + ; +in + respond "OK" { + Content-type = "image/svg+xml"; + } (builtins.readFile "${depot.tvix.docs.svg}/component-flow.svg") diff --git a/web/bubblegum/examples/hello.nix b/web/bubblegum/examples/hello.nix new file mode 100644 index 0000000000..881426bd12 --- /dev/null +++ b/web/bubblegum/examples/hello.nix @@ -0,0 +1,80 @@ +{ depot, ... }: + +let + inherit (depot) + lib + ; + + inherit (depot.web.bubblegum) + pathInfo + respond + absolutePath + ; + + routes = { + "/" = { + status = "OK"; + title = "index"; + content = '' + Hello World! + ''; + }; + "/clock" = { + status = "OK"; + title = "clock"; + content = '' + It is ${toString builtins.currentTime}s since 1970-01-01 00:00 UTC. + ''; + }; + "/coffee" = { + status = "I'm a teapot"; + title = "coffee"; + content = '' + No coffee, I'm afraid + ''; + }; + }; + + notFound = { + status = "Not Found"; + title = "404"; + content = '' + This page doesn't exist. + ''; + }; + + navigation = + lib.concatStrings (lib.mapAttrsToList + (p: v: "
  • ${v.title}
    • ") + routes); + + template = { title, content, ... }: '' + + + + + ${title} + + + +
    +

    //web/bubblegum

    +

    example app

    +
    +
    + +
    +
    +

    ${content}

    +
    + + ''; + + response = routes."${pathInfo}" or notFound; + +in + respond response.status { + "Content-type" = "text/html"; + } (template response) diff --git a/web/bubblegum/examples/posts/2021-04-01-hello.html b/web/bubblegum/examples/posts/2021-04-01-hello.html new file mode 100644 index 0000000000..3c58be7953 --- /dev/null +++ b/web/bubblegum/examples/posts/2021-04-01-hello.html @@ -0,0 +1,3 @@ +

    + This is it, the peak of cursed. +

    diff --git a/web/bubblegum/examples/posts/2021-04-02-a second post.html b/web/bubblegum/examples/posts/2021-04-02-a second post.html new file mode 100644 index 0000000000..050586c302 --- /dev/null +++ b/web/bubblegum/examples/posts/2021-04-02-a second post.html @@ -0,0 +1,7 @@ +

    +

    +

    -- cgit 1.4.1