diff options
Diffstat (limited to 'users/Profpatsch')
45 files changed, 4501 insertions, 0 deletions
diff --git a/users/Profpatsch/OWNERS b/users/Profpatsch/OWNERS new file mode 100644 index 000000000000..5a73d4c3a1fd --- /dev/null +++ b/users/Profpatsch/OWNERS @@ -0,0 +1,4 @@ +inherited: false +owners: + - Profpatsch + - sterni diff --git a/users/Profpatsch/advent-of-code/2020/01/main.py b/users/Profpatsch/advent-of-code/2020/01/main.py new file mode 100644 index 000000000000..e636017a54d5 --- /dev/null +++ b/users/Profpatsch/advent-of-code/2020/01/main.py @@ -0,0 +1,22 @@ +import sys + +l = [] +with open('./input', 'r') as f: + for line in f: + l.append(int(line)) + +s = set(l) + +res=None +for el in s: + for el2 in s: + if (2020-(el+el2)) in s: + res=(el, el2, 2020-(el+el2)) + break + +if res == None: + sys.exit("could not find a number that adds to 2020") + +print(res) + +print(res[0] * res[1] * res[2]) diff --git a/users/Profpatsch/advent-of-code/2020/02/main.py b/users/Profpatsch/advent-of-code/2020/02/main.py new file mode 100644 index 000000000000..e3b27c382a21 --- /dev/null +++ b/users/Profpatsch/advent-of-code/2020/02/main.py @@ -0,0 +1,77 @@ +import sys + +def parse(line): + a = line.split(sep=" ", maxsplit=1) + assert len(a) == 2 + fromto = a[0].split(sep="-") + assert len(fromto) == 2 + (from_, to) = (int(fromto[0]), int(fromto[1])) + charpass = a[1].split(sep=": ") + assert len(charpass) == 2 + char = charpass[0] + assert len(char) == 1 + pass_ = charpass[1] + assert pass_.endswith("\n") + pass_ = pass_[:-1] + return { + "from": from_, + "to": to, + "char": char, + "pass": pass_ + } + +def char_in_pass(char, pass_): + return pass_.count(char) + +def validate_01(entry): + no = char_in_pass(entry["char"], entry["pass"]) + if no < entry["from"]: + return { "too-small": entry } + elif no > entry["to"]: + return { "too-big": entry } + else: + return { "ok": entry } + +def char_at_pos(char, pos, pass_): + assert pos <= len(pass_) + return pass_[pos-1] == char + +def validate_02(entry): + one = char_at_pos(entry["char"], entry["from"], entry["pass"]) + two = char_at_pos(entry["char"], entry["to"], entry["pass"]) + if one and two: + return { "both": entry } + elif one: + return { "one": entry } + elif two: + return { "two": entry } + else: + return { "none": entry } + + +res01 = [] +res02 = [] +with open("./input", 'r') as f: + for line in f: + p = parse(line) + res01.append(validate_01(p)) + res02.append(validate_02(p)) + +count01=0 +for r in res01: + print(r) + if r.get("ok", False): + count01=count01+1 + +count02=0 +for r in res02: + print(r) + if r.get("one", False): + count02=count02+1 + elif r.get("two", False): + count02=count02+1 + else: + pass + +print("count 1: {}".format(count01)) +print("count 2: {}".format(count02)) diff --git a/users/Profpatsch/advent-of-code/2020/03/main.py b/users/Profpatsch/advent-of-code/2020/03/main.py new file mode 100644 index 000000000000..4d6baf946c3e --- /dev/null +++ b/users/Profpatsch/advent-of-code/2020/03/main.py @@ -0,0 +1,66 @@ +import itertools +import math + +def tree_line(init): + return { + "init-len": len(init), + "known": '', + "rest": itertools.repeat(init) + } + +def tree_line_at(pos, tree_line): + needed = (pos + 1) - len(tree_line["known"]) + # internally advance the tree line to the position requested + if needed > 0: + tree_line["known"] = tree_line["known"] \ + + ''.join( + itertools.islice( + tree_line["rest"], + 1+math.floor(needed / tree_line["init-len"]))) + # print(tree_line) + return tree_line["known"][pos] == '#' + +def tree_at(linepos, pos, trees): + return tree_line_at(pos, trees[linepos]) + +def slope_positions(trees, right, down): + line = 0 + pos = 0 + while line < len(trees): + yield (line, pos) + line = line + down + pos = pos + right + +trees = [] +with open("./input", 'r') as f: + for line in f: + line = line.rstrip() + trees.append(tree_line(line)) + +# print(list(itertools.islice(trees[0], 5))) +# print(list(map( +# lambda x: tree_at(0, x, trees), +# range(100) +# ))) +# print(list(slope_positions(trees, right=3, down=1))) + +def count_slope_positions(trees, slope): + count = 0 + for (line, pos) in slope: + if tree_at(line, pos, trees): + count = count + 1 + return count + +print( + count_slope_positions(trees, slope_positions(trees, right=1, down=1)) + * + count_slope_positions(trees, slope_positions(trees, right=3, down=1)) + * + count_slope_positions(trees, slope_positions(trees, right=5, down=1)) + * + count_slope_positions(trees, slope_positions(trees, right=7, down=1)) + * + count_slope_positions(trees, slope_positions(trees, right=1, down=2)) +) + +# I realized I could have just used a modulo instead … diff --git a/users/Profpatsch/advent-of-code/2020/04/main.py b/users/Profpatsch/advent-of-code/2020/04/main.py new file mode 100644 index 000000000000..36bbed7146d6 --- /dev/null +++ b/users/Profpatsch/advent-of-code/2020/04/main.py @@ -0,0 +1,104 @@ +import sys +import itertools +import re +import pprint + +def get_entry(fd): + def to_dict(keyval): + res = {} + for (k, v) in keyval: + assert k not in res + res[k] = v + return res + + res = [] + for line in fd: + if line == "\n": + yield to_dict(res) + res = [] + else: + line = line.rstrip() + items = line.split(" ") + for i in items: + res.append(i.split(":", maxsplit=2)) + +def val_hgt(hgt): + m = re.fullmatch(r'([0-9]+)(cm|in)', hgt) + if m: + (i, what) = m.group(1,2) + i = int(i) + if what == "cm": + return i >= 150 and i <= 193 + elif what == "in": + return i >= 59 and i <= 76 + else: + return False + +required_fields = [ + { "name": "byr", + "check": lambda s: int(s) >= 1920 and int(s) <= 2002 + }, + { "name": "iyr", + "check": lambda s: int(s) >= 2010 and int(s) <= 2020 + }, + { "name": "eyr", + "check": lambda s: int(s) >= 2020 and int(s) <= 2030, + }, + { "name": "hgt", + "check": lambda s: val_hgt(s) + }, + { "name": "hcl", + "check": lambda s: re.fullmatch(r'#[0-9a-f]{6}', s) + }, + { "name": "ecl", + "check": lambda s: re.fullmatch(r'amb|blu|brn|gry|grn|hzl|oth', s) + }, + { "name": "pid", + "check": lambda s: re.fullmatch(r'[0-9]{9}', s) + }, + # we should treat it as not required + # "cid" +] + +required_dict = {} +for f in required_fields: + required_dict[f["name"]] = f + +def validate(keyval): + if keyval[0] not in required_dict: + return { "ok": keyval } + if required_dict[keyval[0]]["check"](keyval[1]): + return { "ok": keyval } + else: + return { "validation": keyval } + +def all_fields(entry): + missing = [] + for r in required_dict: + if r not in e: + missing.append(r) + if missing == []: + return { "ok": entry } + else: + return { "missing": missing } + +count=0 +for e in get_entry(sys.stdin): + a = all_fields(e) + if a.get("ok", False): + res = {} + bad = False + for keyval in e.items(): + r = validate(keyval) + if r.get("validation", False): + bad = True + res[keyval[0]] = r + if bad: + pprint.pprint({ "validation": res }) + else: + pprint.pprint({ "ok": e }) + count = count+1 + else: + pprint.pprint(a) + +print(count) diff --git a/users/Profpatsch/arglib/netencode.nix b/users/Profpatsch/arglib/netencode.nix new file mode 100644 index 000000000000..50f4c11c2d8f --- /dev/null +++ b/users/Profpatsch/arglib/netencode.nix @@ -0,0 +1,40 @@ +{ depot, pkgs, lib, ... }: + +let + netencode = { + rust = depot.nix.writers.rustSimpleLib { + name = "arglib-netencode"; + dependencies = [ + depot.users.Profpatsch.execline.exec-helpers + depot.users.Profpatsch.netencode.netencode-rs + ]; + } '' + extern crate netencode; + extern crate exec_helpers; + + use netencode::{T}; + use std::os::unix::ffi::OsStrExt; + + pub fn arglib_netencode(prog_name: &str, env: Option<&std::ffi::OsStr>) -> T { + let env = match env { + None => std::ffi::OsStr::from_bytes("ARGLIB_NETENCODE".as_bytes()), + Some(a) => a + }; + let t = match std::env::var_os(env) { + None => exec_helpers::die_user_error(prog_name, format!("could not read args, envvar {} not set", env.to_string_lossy())), + // TODO: good error handling for the different parser errors + Some(soup) => match netencode::parse::t_t(soup.as_bytes()) { + Ok((remainder, t)) => match remainder.is_empty() { + true => t, + false => exec_helpers::die_environment_problem(prog_name, format!("arglib: there was some unparsed bytes remaining: {:?}", remainder)) + }, + Err(err) => exec_helpers::die_environment_problem(prog_name, format!("arglib parsing error: {:?}", err)) + } + }; + std::env::remove_var(env); + t + } + ''; + }; + +in depot.nix.readTree.drvTargets netencode diff --git a/users/Profpatsch/atomically-write.nix b/users/Profpatsch/atomically-write.nix new file mode 100644 index 000000000000..d5039d3e46b9 --- /dev/null +++ b/users/Profpatsch/atomically-write.nix @@ -0,0 +1,28 @@ +{ depot, pkgs, ... }: +# Atomically write a file (just `>` redirection in bash +# empties a file even if the command crashes). +# +# Maybe there is an existing tool for that? +# But it’s easy enough to implement. +# +# Example: +# atomically-write +# ./to +# echo "foo" +# +# will atomically write the string "foo" into ./to +let + atomically-write = pkgs.writers.writeDash "atomically-write" '' + set -e + to=$1 + shift + # assumes that the tempfile is on the same file system, (or in memory) + # for the `mv` at the end to be more-or-less atomic. + tmp=$(${pkgs.coreutils}/bin/mktemp -d) + trap 'rm -r "$tmp"' EXIT + "$@" \ + > "$tmp/out" + mv "$tmp/out" "$to" + ''; + +in atomically-write diff --git a/users/Profpatsch/blog/default.nix b/users/Profpatsch/blog/default.nix new file mode 100644 index 000000000000..6ac3c3eb5104 --- /dev/null +++ b/users/Profpatsch/blog/default.nix @@ -0,0 +1,373 @@ +{ depot, pkgs, lib, ... }: + +let + bins = depot.nix.getBins pkgs.lowdown [ "lowdown" ] + // depot.nix.getBins pkgs.cdb [ "cdbget" "cdbmake" "cdbdump" ] + // depot.nix.getBins pkgs.coreutils [ "mv" "cat" "printf" "test" ] + // depot.nix.getBins pkgs.s6-networking [ "s6-tcpserver" ] + // depot.nix.getBins pkgs.time [ "time" ] + ; + + # / + # TODO: use + toplevel = [ + { + route = [ "notes" ]; + name = "Notes"; + page = {cssFile}: router cssFile; + } + { + route = [ "projects" ]; + name = "Projects"; + # page = projects; + } + ]; + + # /notes/* + notes = [ + { + route = [ "notes" "preventing-oom" ]; + name = "Preventing out-of-memory (OOM) errors on Linux"; + page = {cssFile}: markdownToHtml { + name = "preventing-oom"; + markdown = ./notes/preventing-oom.md; + inherit cssFile; + }; + } + { + route = [ "notes" "rust-string-conversions" ]; + name = "Converting between different String types in Rust"; + page = {cssFile}: markdownToHtml { + name = "rust-string-conversions"; + markdown = ./notes/rust-string-conversions.md; + inherit cssFile; + }; + } + ]; + + projects = [ + { + name = "lorri"; + description = "<code>nix-shell</code> replacement for projects"; + link = "https://github.com/nix-community/lorri"; + } + { + name = "netencode"; + description = ''A human-readble nested data exchange format inspired by <a href="https://en.wikipedia.org/wiki/Netstring">netstrings</a> and <a href="https://en.wikipedia.org/wiki/Bencode">bencode</a>.''; + link = depotCgitLink { relativePath = "users/Profpatsch/netencode/README.md"; }; + } + { + name = "yarn2nix"; + description = ''nix dependency generator for the <a href="https://yarnpkg.com/"><code>yarn</code> Javascript package manager</a>''; + link = "https://github.com/Profpatsch/yarn2nix"; + } + ]; + + posts = [ + { + date = "2017-05-04"; + title = "Ligature Emulation in Emacs"; + subtitle = "It’s not pretty, but the results are"; + description = "How to set up ligatures using <code>prettify-symbols-mode</code> and the Hasklig/FiraCode fonts."; + page = {cssFile}: markdownToHtml { + name = "2017-05-04-ligature-emluation-in-emacs"; + markdown = ./posts/2017-05-04-ligature-emulation-in-emacs.md; + inherit cssFile; + }; + route = [ "posts" "2017-05-04-ligature-emluation-in-emacs" ]; + tags = ["emacs"]; + } + ]; + + # convert a markdown file to html via lowdown + markdownToHtml = { + name, + # the file to convert + markdown, + # css file to add to the final result, as { route } + cssFile + }: + depot.nix.runExecline "${name}.html" {} ([ + "importas" "out" "out" + (depot.users.Profpatsch.lib.debugExec "") + bins.lowdown + "-s" "-Thtml" + ] ++ + (lib.optional (cssFile != null) (["-M" "css=${mkRoute cssFile.route}"])) + ++ [ + "-o" "$out" + markdown + ]); + + # takes a { route … } attrset and converts the route lists to an absolute path + fullRoute = attrs: lib.pipe attrs [ + (map (x@{route, ...}: x // { route = mkRoute route; })) + ]; + + # a cdb from route to a netencoded version of data for each route + router = cssFile: lib.pipe (notes ++ posts) [ + (map (r: with depot.users.Profpatsch.lens; + lib.pipe r [ + (over (field "route") mkRoute) + (over (field "page") (_ { inherit cssFile; })) + ])) + (map (x: { + name = x.route; + value = depot.users.Profpatsch.netencode.gen.dwim x; + })) + lib.listToAttrs + (cdbMake "router") + ]; + + # Create a link to the given source file/directory, given the relative path in the depot repo. + # Checks that the file exists at evaluation time. + depotCgitLink = { + # relative path from the depot root (without leading /). + relativePath + }: + assert + (lib.assertMsg + (builtins.pathExists (depot.path + "/" + relativePath)) + "depotCgitLink: path /${relativePath} does not exist in depot"); + "https://code.tvl.fyi/tree/${relativePath}"; + + # look up a route by path ($1) + router-lookup = cssFile: depot.nix.writeExecline "router-lookup" { readNArgs = 1; } [ + cdbLookup (router cssFile) "$1" + ]; + + runExeclineStdout = name: args: cmd: depot.nix.runExecline name args ([ + "importas" "-ui" "out" "out" + "redirfd" "-w" "1" "$out" + ] ++ cmd); + + notes-index-html = + let o = fullRoute notes; + in '' + <ul> + ${scope o (o: '' + <li><a href="${str o.route}">${esc o.name}</a></li> + '')} + </ul> + ''; + + notes-index = pkgs.writeText "notes-index.html" notes-index-html; + + # A simple mustache-inspired string interpolation combinator + # that takes an object and a template (a function from o to string) + # and returns a string. + scope = o: tpl: + if builtins.typeOf o == "list" then + lib.concatMapStringsSep "\n" tpl o + else if builtins.typeOf o == "set" then + tpl o + else throw "${lib.generators.toPretty {} o} not allowed in template"; + + # string-escape html (TODO) + str = s: s; + # html-escape (TODO) + esc = s: s; + html = s: s; + + projects-index-html = + let o = projects; + in '' + <dl> + ${scope o (o: '' + <dt><a href="${str o.link}">${esc o.name}</a></dt> + <dd>${html o.description}</dd> + '')} + </dl> + ''; + + projects-index = pkgs.writeText "projects-index.html" projects-index-html; + + posts-index-html = + let o = fullRoute posts; + in '' + <dl> + ${scope o (o: '' + <dt>${str o.date} <a href="${str o.route}">${esc o.title}</a></dt> + <dd>${html o.description}</dd> + '')} + </dl> + ''; + + posts-index = pkgs.writeText "projects-index.html" posts-index-html; + + arglibNetencode = val: depot.nix.writeExecline "arglib-netencode" { } [ + "export" "ARGLIB_NETENCODE" (depot.users.Profpatsch.netencode.gen.dwim val) + "$@" + ]; + + # A simple http server that serves the site. Yes, it’s horrible. + site-server = { cssFile, port }: depot.nix.writeExecline "blog-server" {} [ + (depot.users.Profpatsch.lib.runInEmptyEnv [ "PATH" ]) + bins.s6-tcpserver "127.0.0.1" port + bins.time "--format=time: %es" "--" + runOr return400 + "pipeline" [ + (arglibNetencode { + what = "request"; + }) + depot.users.Profpatsch.read-http + ] + depot.users.Profpatsch.netencode.record-splice-env + runOr return500 + "importas" "-i" "path" "path" + "if" [ depot.tools.eprintf "GET \${path}\n" ] + runOr return404 + "backtick" "-ni" "TEMPLATE_DATA" [ + # TODO: factor this out of here, this is routing not serving + "ifelse" [ bins.test "$path" "=" "/notes" ] + [ "export" "content-type" "text/html" + "export" "serve-file" notes-index + depot.users.Profpatsch.netencode.env-splice-record + ] + "ifelse" [ bins.test "$path" "=" "/projects" ] + [ "export" "content-type" "text/html" + "export" "serve-file" projects-index + depot.users.Profpatsch.netencode.env-splice-record + ] + "ifelse" [ bins.test "$path" "=" "/posts" ] + [ "export" "content-type" "text/html" + "export" "serve-file" posts-index + depot.users.Profpatsch.netencode.env-splice-record + ] + # TODO: ignore potential query arguments. See 404 message + "pipeline" [ (router-lookup cssFile) "$path" ] + depot.users.Profpatsch.netencode.record-splice-env + "importas" "-ui" "page" "page" + "export" "content-type" "text/html" + "export" "serve-file" "$page" + depot.users.Profpatsch.netencode.env-splice-record + ] + runOr return500 + "if" [ + "pipeline" [ bins.printf '' + HTTP/1.1 200 OK + Content-Type: {{{content-type}}}; charset=UTF-8 + Connection: close + + '' ] + depot.users.Profpatsch.netencode.netencode-mustache + ] + "pipeline" [ "importas" "t" "TEMPLATE_DATA" bins.printf "%s" "$t" ] + depot.users.Profpatsch.netencode.record-splice-env + "importas" "-ui" "serve-file" "serve-file" + bins.cat "$serve-file" + ]; + + # run argv or $1 if argv returns a failure status code. + runOr = depot.nix.writeExecline "run-or" { readNArgs = 1; } [ + "foreground" [ "$@" ] + "importas" "?" "?" + "ifelse" [ bins.test "$?" "-eq" "0" ] + [] + "if" [ depot.tools.eprintf "runOr: exited \${?}, running \${1}\n" ] + "$1" + ]; + + return400 = depot.nix.writeExecline "return400" {} [ + bins.printf "%s" '' + HTTP/1.1 400 Bad Request + Content-Type: text/plain; charset=UTF-8 + Connection: close + + '' + ]; + + return404 = depot.nix.writeExecline "return404" {} [ + bins.printf "%s" '' + HTTP/1.1 404 Not Found + Content-Type: text/plain; charset=UTF-8 + Connection: close + + This page doesn’t exist! Query arguments are not handled at the moment. + '' + ]; + + return500 = depot.nix.writeExecline "return500" {} [ + bins.printf "%s" '' + HTTP/1.1 500 Internal Server Error + Content-Type: text/plain; charset=UTF-8 + Connection: close + + Encountered an internal server error. Please try again. + '' + ]; + + capture-stdin = depot.nix.writers.rustSimple { + name = "capture-stdin"; + dependencies = [ depot.users.Profpatsch.execline.exec-helpers ]; + } '' + extern crate exec_helpers; + use std::io::Read; + fn main() { + let (args, prog) = exec_helpers::args_for_exec("capture-stdin", 1); + let valname = &args[1]; + let mut v : Vec<u8> = vec![]; + std::io::stdin().lock().read_to_end(&mut v).unwrap(); + exec_helpers::exec_into_args("capture-stdin", prog, vec![(valname, v)]); + } + ''; + + # go from a list of path elements to an absolute route string + mkRoute = route: "/" + lib.concatMapStringsSep "/" urlencodeAscii route; + + # urlencodes, but only ASCII characters + # https://en.wikipedia.org/wiki/Percent-encoding + urlencodeAscii = urlPiece: + let + raw = [ "!" "#" "$" "%" "&" "'" "(" ")" "*" "+" "," "/" ":" ";" "=" "?" "@" "[" "]" ]; + enc = [ "%21" "%23" "%24" "%25" "%26" "%27" "%28" "%29" "%2A" "%2B" "%2C" "%2F" "%3A" "%3B" "%3D" "%3F" "%40" "%5B" "%5D" ]; + rest = [ "A" "B" "C" "D" "E" "F" "G" "H" "I" "J" "K" "L" "M" "N" "O" "P" "Q" "R" "S" "T" "U" "V" "W" "X" "Y" "Z" "a" "b" "c" "d" "e" "f" "g" "h" "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t" "u" "v" "w" "x" "y" "z" "0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "-" "_" "." "~" ]; + in + assert lib.assertMsg (lib.all (c: builtins.elem c (raw ++ rest)) (lib.stringToCharacters urlPiece)) + "urlencodeAscii: the urlPiece must only contain valid url ASCII characters, was: ${urlPiece}"; + builtins.replaceStrings raw enc urlPiece; + + + # create a cdb record entry, as required by the cdbmake tool + cdbRecord = key: val: + "+${toString (builtins.stringLength key)},${toString (builtins.stringLength val)}:" + + "${key}->${val}\n"; + + # create a full cdbmake input from an attribute set of keys to values (strings) + cdbRecords = + with depot.nix.yants; + defun [ (attrs (either drv string)) string ] + (attrs: + (lib.concatStrings (lib.mapAttrsToList cdbRecord attrs)) + "\n"); + + # run cdbmake on a list of key/value pairs (strings + cdbMake = name: attrs: depot.nix.runExecline "${name}.cdb" { + stdin = cdbRecords attrs; + } [ + "importas" "out" "out" + depot.users.Profpatsch.lib.eprint-stdin + "if" [ bins.cdbmake "db" "tmp" ] + bins.mv "db" "$out" + ]; + + # look up a key ($2) in the given cdb ($1) + cdbLookup = depot.nix.writeExecline "cdb-lookup" { readNArgs = 2; } [ + # cdb ($1) on stdin + "redirfd" "-r" "0" "$1" + # key ($2) lookup + bins.cdbget "$2" + ]; + +in depot.nix.readTree.drvTargets { + inherit + router + depotCgitLink + site-server + notes-index + notes-index-html + projects-index + projects-index-html + posts-index-html + ; + +} diff --git a/users/Profpatsch/blog/notes/preventing-oom.md b/users/Profpatsch/blog/notes/preventing-oom.md new file mode 100644 index 000000000000..59ea4f747700 --- /dev/null +++ b/users/Profpatsch/blog/notes/preventing-oom.md @@ -0,0 +1,33 @@ +tags: linux +date: 2020-01-25 +certainty: likely +status: initial +title: Preventing out-of-memory (OOM) errors on Linux + +# Preventing out-of-memory (OOM) errors on Linux + +I’ve been running out of memory more and more often lately. I don’t use any swap space because I am of the opinion that 16GB of memory should be sufficient for most daily and professional tasks. Which is generally true, however sometimes I have a runaway filling my memory. Emacs is very good at doing this for example, prone to filling your RAM when you open json files with very long lines. + +In theory, the kernel OOM killer should come in and save the day, but the Linux OOM killer is notorious for being extremely … conservative. It will try to free every internal structure it can before even thinking about touching any userspace processes. At that point, the desktop usually stopped responding minutes ago. + +Luckily the kernel provides memory statistics for the whole system, as well as single process, and the [`earlyoom`](https://github.com/rfjakob/earlyoom) tool uses those to keep memory usage under a certain limit. It will start killing processes, “heaviest” first, until the given upper memory limit is satisfied again. + +On NixOS, I set: + +```nix +{ + services.earlyoom = { + enable = true; + freeMemThreshold = 5; # <%5 free + }; +} +``` + +and after activation, this simple test shows whether the daemon is working: + +```shell +$ tail /dev/zero +fish: “tail /dev/zero” terminated by signal SIGTERM (Polite quit request) +``` + +`tail /dev/zero` searches for the last line of the file `/dev/zero`, and since it cannot know that there is no next line and no end to the stream of `\0` this file produces, it will fill the RAM as quickly as physically possible. Before it can fill it completely, `earlyoom` recognizes that the limit was breached, singles out the `tail` command as the process using the most amount of memory, and sends it a `SIGTERM`. diff --git a/users/Profpatsch/blog/notes/rust-string-conversions.md b/users/Profpatsch/blog/notes/rust-string-conversions.md new file mode 100644 index 000000000000..99071ef9d370 --- /dev/null +++ b/users/Profpatsch/blog/notes/rust-string-conversions.md @@ -0,0 +1,53 @@ +# Converting between different String types in Rust + +``` +let s: String = ... +let st: &str = ... +let u: &[u8] = ... +let b: [u8; 3] = b"foo" +let v: Vec<u8> = ... +let os: OsString = ... +let ost: OsStr = ... + +From To Use Comment +---- -- --- ------- +&str -> String String::from(st) +&str -> &[u8] st.as_bytes() +&str -> Vec<u8> st.as_bytes().to_owned() via &[u8] +&str -> &OsStr OsStr::new(st) + +String -> &str &s alt. s.as_str() +String -> &[u8] s.as_bytes() +String -> Vec<u8> s.into_bytes() +String -> OsString OsString::from(s) + +&[u8] -> &str str::from_utf8(u).unwrap() +&[u8] -> String String::from_utf8(u).unwrap() +&[u8] -> Vec<u8> u.to_owned() +&[u8] -> &OsStr OsStr::from_bytes(u) use std::os::unix::ffi::OsStrExt; + +[u8; 3] -> &[u8] &b[..] byte literal +[u8; 3] -> &[u8] "foo".as_bytes() alternative via utf8 literal + +Vec<u8> -> &str str::from_utf8(&v).unwrap() via &[u8] +Vec<u8> -> String String::from_utf8(v) +Vec<u8> -> &[u8] &v +Vec<u8> -> OsString OsString::from_vec(v) use std::os::unix::ffi::OsStringExt; + +&OsStr -> &str ost.to_str().unwrap() +&OsStr -> String ost.to_os_string().into_string() via OsString + .unwrap() +&OsStr -> Cow<str> ost.to_string_lossy() Unicode replacement characters +&OsStr -> OsString ost.to_os_string() +&OsStr -> &[u8] ost.as_bytes() use std::os::unix::ffi::OsStringExt; + +OsString -> String os.into_string().unwrap() returns original OsString on failure +OsString -> &str os.to_str().unwrap() +OsString -> &OsStr os.as_os_str() +OsString -> Vec<u8> os.into_vec() use std::os::unix::ffi::OsStringExt; +``` + + +## Source + +Original source is [this document on Pastebin](https://web.archive.org/web/20190710121935/https://pastebin.com/Mhfc6b9i) diff --git a/users/Profpatsch/blog/posts/2017-05-04-ligature-emulation-in-emacs.md b/users/Profpatsch/blog/posts/2017-05-04-ligature-emulation-in-emacs.md new file mode 100644 index 000000000000..ba80888badd8 --- /dev/null +++ b/users/Profpatsch/blog/posts/2017-05-04-ligature-emulation-in-emacs.md @@ -0,0 +1,123 @@ +title: Ligature Emulation in Emacs +date: 2017-05-04 + +Monday was (yet another) +[NixOS hackathon][hackathon] at [OpenLab Augsburg][ola]. +[Maximilian][mhuber] was there and to my amazement +he got working ligatures in his Haskell files in Emacs! Ever since Hasklig +updated its format to use ligatures and private Unicode code points a while ago, +the hack I had used in my config stopped working. + +Encouraged by that I decided to take a look on Tuesday. Long story short, I was +able to [get it working in a pretty satisfying way][done]. + +[hackathon]: https://www.meetup.com/Munich-NixOS-Meetup/events/239077247/ +[mhuber]: https://github.com/maximilianhuber +[ola]: https://openlab-augsburg.de +[done]: https://github.com/i-tu/Hasklig/issues/84#issuecomment-298803495 + +What’s left to do is package it into a module and push to melpa. + + +### elisp still sucks, but it’s bearable, sometimes + +I’m the kind of person who, when trying to fix something elisp related, normally +gives up two hours later and three macro calls deep. Yes, homoiconic, +non-lexically-scoped, self-rewriting code is not exactly my fetish. +This time the task and the library (`prettify-symbols-mode`) were simple enough +for that to not happen. + +Some interesting technical trivia: + +- elisp literal character syntax is `?c`. `?\t` is the tab character +- You join characters by `(string c1 c2 c3 ...)` +- [dash.el][dash] is pretty awesome and does what a functional programmer + expects. Also, Rainbow Dash. +- Hasklig and FiraCode multi-column symbols actually [only occupy one column, on + the far right of the glyph][glyph]. `my-correct-symbol-bounds` fixes emacs’ + rendering in that case. + + +[dash]: https://github.com/magnars/dash.el +[glyph]: https://github.com/tonsky/FiraCode/issues/211#issuecomment-239082368 + + +## Appendix A + +For reference, here’s the complete code as it stands now. Feel free to paste +into your config; let’s make it [MIT][mit]. Maybe link to this site, in case there are +updates. + +[mit]: https://opensource.org/licenses/MIT + +```elisp + (defun my-correct-symbol-bounds (pretty-alist) + "Prepend a TAB character to each symbol in this alist, +this way compose-region called by prettify-symbols-mode +will use the correct width of the symbols +instead of the width measured by char-width." + (mapcar (lambda (el) + (setcdr el (string ?\t (cdr el))) + el) + pretty-alist)) + + (defun my-ligature-list (ligatures codepoint-start) + "Create an alist of strings to replace with +codepoints starting from codepoint-start." + (let ((codepoints (-iterate '1+ codepoint-start (length ligatures)))) + (-zip-pair ligatures codepoints))) + + ; list can be found at https://github.com/i-tu/Hasklig/blob/master/GlyphOrderAndAliasDB#L1588 + (setq my-hasklig-ligatures + (let* ((ligs '("&&" "***" "*>" "\\\\" "||" "|>" "::" + "==" "===" "==>" "=>" "=<<" "!!" ">>" + ">>=" ">>>" ">>-" ">-" "->" "-<" "-<<" + "<*" "<*>" "<|" "<|>" "<$>" "<>" "<-" + "<<" "<<<" "<+>" ".." "..." "++" "+++" + "/=" ":::" ">=>" "->>" "<=>" "<=<" "<->"))) + (my-correct-symbol-bounds (my-ligature-list ligs #Xe100)))) + + ;; nice glyphs for haskell with hasklig + (defun my-set-hasklig-ligatures () + "Add hasklig ligatures for use with prettify-symbols-mode." + (setq prettify-symbols-alist + (append my-hasklig-ligatures prettify-symbols-alist)) + (prettify-symbols-mode)) + + (add-hook 'haskell-mode-hook 'my-set-hasklig-ligatures) +``` + +## Appendix B (Update 1): FiraCode integration + +I also created a mapping for [FiraCode][fira]. You need to grab the [additional +symbol font][symbol] that adds (most) ligatures to the unicode private use area. +Consult your system documentation on how to add it to your font cache. +Next add `"Fira Code"` and `"Fira Code Symbol"` to your font preferences. Symbol +only contains the additional characters, so you need both. + +If you are on NixOS, the font package should be on the main branch shortly, [I +added a package][symbol-pkg]. + +[fira]: https://github.com/tonsky/FiraCode/ +[symbol]: https://github.com/tonsky/FiraCode/issues/211#issuecomment-239058632 +[symbol-pkg]: https://github.com/NixOS/nixpkgs/pull/25517 + +Here’s the mapping adjusted for FiraCode: + +```elisp + (setq my-fira-code-ligatures + (let* ((ligs '("www" "**" "***" "**/" "*>" "*/" "\\\\" "\\\\\\" + "{-" "[]" "::" ":::" ":=" "!!" "!=" "!==" "-}" + "--" "---" "-->" "->" "->>" "-<" "-<<" "-~" + "#{" "#[" "##" "###" "####" "#(" "#?" "#_" "#_(" + ".-" ".=" ".." "..<" "..." "?=" "??" ";;" "/*" + "/**" "/=" "/==" "/>" "//" "///" "&&" "||" "||=" + "|=" "|>" "^=" "$>" "++" "+++" "+>" "=:=" "==" + "===" "==>" "=>" "=>>" "<=" "=<<" "=/=" ">-" ">=" + ">=>" ">>" ">>-" ">>=" ">>>" "<*" "<*>" "<|" "<|>" + "<$" "<$>" "<!--" "<-" "<--" "<->" "<+" "<+>" "<=" + "<==" "<=>" "<=<" "<>" "<<" "<<-" "<<=" "<<<" "<~" + "<~~" "</" "</>" "~@" "~-" "~=" "~>" "~~" "~~>" "%%" + "x" ":" "+" "+" "*"))) + (my-correct-symbol-bounds (my-ligature-list ligs #Xe100)))) +``` diff --git a/users/Profpatsch/cdb.nix b/users/Profpatsch/cdb.nix new file mode 100644 index 000000000000..8cfaa3ea7ac6 --- /dev/null +++ b/users/Profpatsch/cdb.nix @@ -0,0 +1,91 @@ +{ depot, pkgs, ... }: + +let + cdbListToNetencode = depot.nix.writers.rustSimple { + name = "cdb-list-to-netencode"; + dependencies = [ + depot.third_party.rust-crates.nom + depot.users.Profpatsch.execline.exec-helpers + depot.users.Profpatsch.netencode.netencode-rs + ]; + } '' + extern crate nom; + extern crate exec_helpers; + extern crate netencode; + use std::collections::HashMap; + use std::io::BufRead; + use nom::{IResult}; + use nom::sequence::{tuple}; + use nom::bytes::complete::{tag, take}; + use nom::character::complete::{digit1, char}; + use nom::error::{context, ErrorKind, ParseError}; + use nom::combinator::{map_res}; + use netencode::{T, Tag}; + + fn usize_t(s: &[u8]) -> IResult<&[u8], usize> { + context( + "usize", + map_res( + map_res(digit1, |n| std::str::from_utf8(n)), + |s| s.parse::<usize>()) + )(s) + } + + fn parse_cdb_record(s: &[u8]) -> IResult<&[u8], (&[u8], &[u8])> { + let (s, (_, klen, _, vlen, _)) = tuple(( + char('+'), + usize_t, + char(','), + usize_t, + char(':') + ))(s)?; + let (s, (key, _, val)) = tuple(( + take(klen), + tag("->"), + take(vlen), + ))(s)?; + Ok((s, (key, val))) + } + + fn main() { + let mut res = vec![]; + let stdin = std::io::stdin(); + let mut lines = stdin.lock().split(b'\n'); + loop { + match lines.next() { + None => exec_helpers::die_user_error("cdb-list-to-netencode", "stdin ended but we didn’t receive the empty line to signify the end of the cdbdump input!"), + Some(Err(err)) => exec_helpers::die_temporary("cdb-list-to-netencode", format!("could not read from stdin: {}", err)), + Some(Ok(line)) => + if &line == b"" { + // the cdbdump input ends after an empty line (double \n) + break; + } else { + match parse_cdb_record(&line) { + Ok((b"", (key, val))) => { + let (key, val) = match + std::str::from_utf8(key) + .and_then(|k| std::str::from_utf8(val).map(|v| (k, v))) { + Ok((key, val)) => (key.to_owned(), val.to_owned()), + Err(err) => exec_helpers::die_user_error("cdb-list-to-netencode", format!("cannot decode line {:?}, we only support utf8-encoded key/values pairs for now: {}", String::from_utf8_lossy(&line), err)), + }; + let _ = res.push((key, val)); + }, + Ok((rest, _)) => exec_helpers::die_user_error("cdb-list-to-netencode", format!("could not decode record line {:?}, had some trailing bytes", String::from_utf8_lossy(&line))), + Err(err) => exec_helpers::die_user_error("cdb-list-to-netencode", format!("could not decode record line {:?}: {:?}", String::from_utf8_lossy(&line), err)), + } + } + } + } + let list = T::List(res.into_iter().map( + |(k, v)| T::Record(vec![(String::from("key"), T::Text(k)), (String::from("val"), T::Text(v))].into_iter().collect()) + ).collect()); + netencode::encode(&mut std::io::stdout(), &list.to_u()); + } + + ''; + +in { + inherit + cdbListToNetencode + ; +} diff --git a/users/Profpatsch/emacs-tree-sitter-move/default.nix b/users/Profpatsch/emacs-tree-sitter-move/default.nix new file mode 100644 index 000000000000..fdc059c089b6 --- /dev/null +++ b/users/Profpatsch/emacs-tree-sitter-move/default.nix @@ -0,0 +1,3 @@ +# nothing yet (TODO: expose shell & tool) +{...}: +{} diff --git a/users/Profpatsch/emacs-tree-sitter-move/shell.nix b/users/Profpatsch/emacs-tree-sitter-move/shell.nix new file mode 100644 index 000000000000..81d622ac73e5 --- /dev/null +++ b/users/Profpatsch/emacs-tree-sitter-move/shell.nix @@ -0,0 +1,16 @@ +{ pkgs ? import ../../../third_party {}, ... }: +let + inherit (pkgs) lib; + + treeSitterGrammars = pkgs.runCommandLocal "grammars" {} '' + mkdir -p $out/bin + ${lib.concatStringsSep "\n" + (lib.mapAttrsToList (name: src: "ln -s ${src}/parser $out/bin/${name}.so") pkgs.tree-sitter.builtGrammars)}; + ''; + +in pkgs.mkShell { + buildInputs = [ + pkgs.tree-sitter.builtGrammars.python + ]; + TREE_SITTER_GRAMMAR_DIR = treeSitterGrammars; +} diff --git a/users/Profpatsch/emacs-tree-sitter-move/test.json b/users/Profpatsch/emacs-tree-sitter-move/test.json new file mode 100644 index 000000000000..d9f8075976d6 --- /dev/null +++ b/users/Profpatsch/emacs-tree-sitter-move/test.json @@ -0,0 +1,14 @@ +{ + "foo": { + "x": [ 1, 2, 3, 4 ], + "bar": "test" + }, + "foo": { + "x": [ 1, 2, 3, 4 ], + "bar": "test" + }, + "foo": { + "x": [ 1, 2, 3, 4 ], + "bar": "test" + } +} diff --git a/users/Profpatsch/emacs-tree-sitter-move/test.py b/users/Profpatsch/emacs-tree-sitter-move/test.py new file mode 100644 index 000000000000..0f57bae035da --- /dev/null +++ b/users/Profpatsch/emacs-tree-sitter-move/test.py @@ -0,0 +1,13 @@ +(4 + 5 + 5) + +def foo(a, b, c) + +def bar(a, b): + 4 + 4 + 4 + +[1, 4, 5, 10] + +def foo(): + pass diff --git a/users/Profpatsch/emacs-tree-sitter-move/test.sh b/users/Profpatsch/emacs-tree-sitter-move/test.sh new file mode 100644 index 000000000000..681081f5909d --- /dev/null +++ b/users/Profpatsch/emacs-tree-sitter-move/test.sh @@ -0,0 +1,14 @@ +function foo () { + local x=123 +} + +function bar () { + local x=123 +} + +echo abc def \ + gef gef + +printf \ + "%s\n" \ + haha diff --git a/users/Profpatsch/emacs-tree-sitter-move/tmp.el b/users/Profpatsch/emacs-tree-sitter-move/tmp.el new file mode 100644 index 000000000000..88d13fa45b81 --- /dev/null +++ b/users/Profpatsch/emacs-tree-sitter-move/tmp.el @@ -0,0 +1,28 @@ +(defun tree-sitter-load-from-grammar-dir (grammar-dir sym lang-name) + (tree-sitter-load + sym + (format "%s/bin/%s" + (getenv grammar-dir) + lang-name))) + +(defun tree-sitter-init-tmp-langs (alist) + (mapcar + (lambda (lang) + (pcase-let ((`(,name ,sym ,mode) lang)) + (tree-sitter-load-from-grammar-dir "TREE_SITTER_GRAMMAR_DIR" sym name) + (cons mode sym))) + alist)) + + +(setq tree-sitter-major-mode-language-alist + (tree-sitter-init-tmp-langs + '(("python" python python-mode) + ("json" json js-mode) + ("bash" bash sh-mode) + ))) + +(define-key evil-normal-state-map (kbd "C-.") #'tree-sitter-move-reset) +(define-key evil-normal-state-map (kbd "C-<right>") #'tree-sitter-move-right) +(define-key evil-normal-state-map (kbd "C-<left>") #'tree-sitter-move-left) +(define-key evil-normal-state-map (kbd "C-<up>") #'tree-sitter-move-up) +(define-key evil-normal-state-map (kbd "C-<down>") #'tree-sitter-move-down) diff --git a/users/Profpatsch/emacs-tree-sitter-move/tree-sitter-move.el b/users/Profpatsch/emacs-tree-sitter-move/tree-sitter-move.el new file mode 100644 index 000000000000..907e1e4081bc --- /dev/null +++ b/users/Profpatsch/emacs-tree-sitter-move/tree-sitter-move.el @@ -0,0 +1,139 @@ +;; this is not an actual cursor, just a node. +;; It’s not super efficient, but cursors can’t be *set* to an arbitrary +;; subnode, because they can’t access the parent otherwise. +;; We’d need a way to reset the cursor and walk down to the node?! +(defvar-local tree-sitter-move--cursor nil + "the buffer-local cursor used for movement") + +(defvar-local tree-sitter-move--debug-overlay nil + "an overlay used to visually display the region currently marked by the cursor") + +;;;;; TODO: should everything use named nodes? Only some things? +;;;;; maybe there should be a pair of functions for everything? +;;;;; For now restrict to named nodes. + +(defun tree-sitter-move--setup () + ;; TODO + (progn + ;; TODO: if tree-sitter-mode fails to load, display a better error + (tree-sitter-mode t) + (setq tree-sitter-move--cursor (tsc-root-node tree-sitter-tree)) + (add-variable-watcher + 'tree-sitter-move--cursor + #'tree-sitter-move--debug-overlay-update))) + +(defun tree-sitter-move--debug-overlay-update (sym newval &rest _args) + "variable-watcher to update the debug overlay when the cursor changes" + (let ((start (tsc-node-start-position newval)) + (end (tsc-node-end-position newval))) + (symbol-macrolet ((o tree-sitter-move--debug-overlay)) + (if o + (move-overlay o start end) + (setq o (make-overlay start end)) + (overlay-put o 'face 'highlight) + )))) + +(defun tree-sitter-move--debug-overlay-teardown () + "Turn of the overlay visibility and delete the overlay object" + (when tree-sitter-move--debug-overlay + (delete-overlay tree-sitter-move--debug-overlay) + (setq tree-sitter-move--debug-overlay nil))) + +(defun tree-sitter-move--teardown () + (setq tree-sitter-move--cursor nil) + (tree-sitter-move--debug-overlay-teardown) + (tree-sitter-mode nil)) + +;; Get the syntax node the cursor is on. +(defun tsc-get-named-node-at-point () + (let ((p (point))) + (tsc-get-named-descendant-for-position-range + (tsc-root-node tree-sitter-tree) p p))) + +;; TODO: is this function necessary? +;; Maybe tree-sitter always guarantees that parents are named? +(defun tsc-get-named-parent (node) + (when-let ((parent (tsc-get-parent node))) + (while (and parent (not (tsc-node-named-p parent))) + (setq parent (tsc-get-parent parent))) + parent)) + +(defun tsc-get-first-named-node-with-siblings-up (node) + "Returns the first 'upwards' node that has siblings. That includes the current + node, so if the given node has siblings, it is returned. Returns nil if there + is no such node until the root" + (when-let ((has-siblings-p + (lambda (parent-node) + (> (tsc-count-named-children parent-node) + 1))) + (cur node) + (parent (tsc-get-named-parent node))) + (while (and parent (not (funcall has-siblings-p parent))) + (setq cur parent) + (setq parent (tsc-get-named-parent cur))) + cur)) + +(defun tree-sitter-move--set-cursor-to-node (node) + (setq tree-sitter-move--cursor node)) + +(defun tree-sitter-move--set-cursor-to-node-at-point () + (tree-sitter-move--set-cursor-to-node (tsc-get-named-node-at-point))) + +(defun tree-sitter-move--move-point-to-node (node) + (set-window-point + (selected-window) + (tsc-node-start-position node))) + + +;; interactive commands (“do what I expect” section) + +(defun tree-sitter-move-reset () + (interactive) + (tree-sitter-move--set-cursor-to-node-at-point)) + +(defun tree-sitter-move-right () + (interactive) + (tree-sitter-move--move-skip-non-sibling-nodes 'tsc-get-next-named-sibling)) + +(defun tree-sitter-move-left () + (interactive) + (tree-sitter-move--move-skip-non-sibling-nodes 'tsc-get-prev-named-sibling)) + +(defun tree-sitter-move-up () + (interactive) + (tree-sitter-move--move-skip-non-sibling-nodes 'tsc-get-parent)) + +;; TODO: does not skip siblings yet, because the skip function only goes up (not down) +(defun tree-sitter-move-down () + (interactive) + (tree-sitter-move--move-if-possible (lambda (n) (tsc-get-nth-named-child n 0)))) + +(defun tree-sitter-move--move-skip-non-sibling-nodes (move-fn) + "Moves to the sidewards next sibling. If the current node does not have siblings, go + upwards until something has siblings and then move to the side (right or left)." + (tree-sitter-move--move-if-possible + (lambda (cur) + (when-let ((with-siblings + (tsc-get-first-named-node-with-siblings-up cur))) + (funcall move-fn with-siblings))))) + +(defun tree-sitter-move--move-if-possible (dir-fn) + (let ((next (funcall dir-fn tree-sitter-move--cursor))) + (when next + (tree-sitter-move--set-cursor-to-node next) + (tree-sitter-move--move-point-to-node next)))) + +; mostly stolen from tree-sitter-mode +;;;###autoload +(define-minor-mode tree-sitter-move-mode + "Minor mode to do cursor movements via tree-sitter" + :init-value nil + :lighter " tree-sitter-move" + (if tree-sitter-move-mode + (tree-sitter--error-protect + (progn + (tree-sitter-move--setup)) + (setq tree-sitter-move-mode nil) + (tree-sitter-move--teardown)) + (lambda ()) + (tree-sitter-move--teardown))) diff --git a/users/Profpatsch/execline/default.nix b/users/Profpatsch/execline/default.nix new file mode 100644 index 000000000000..1f75b97591fc --- /dev/null +++ b/users/Profpatsch/execline/default.nix @@ -0,0 +1,12 @@ +{ depot, pkgs, lib, ... }: + +let + exec-helpers = depot.nix.writers.rustSimpleLib { + name = "exec-helpers"; + } (builtins.readFile ./exec_helpers.rs); + +in depot.nix.readTree.drvTargets { + inherit + exec-helpers + ; +} diff --git a/users/Profpatsch/execline/exec_helpers.rs b/users/Profpatsch/execline/exec_helpers.rs new file mode 100644 index 000000000000..b9e1f5797386 --- /dev/null +++ b/users/Profpatsch/execline/exec_helpers.rs @@ -0,0 +1,113 @@ +use std::os::unix::process::CommandExt; +use std::ffi::OsStr; +use std::os::unix::ffi::{OsStringExt, OsStrExt}; + +pub fn no_args(current_prog_name: &str) -> () { + let mut args = std::env::args_os(); + // remove argv[0] + let _ = args.nth(0); + if args.len() > 0 { + die_user_error(current_prog_name, format!("Expected no arguments, got {:?}", args.collect::<Vec<_>>())) + } +} + +pub fn args(current_prog_name: &str, no_of_positional_args: usize) -> Vec<Vec<u8>> { + let mut args = std::env::args_os(); + // remove argv[0] + let _ = args.nth(0); + if args.len() != no_of_positional_args { + die_user_error(current_prog_name, format!("Expected {} arguments, got {}, namely {:?}", no_of_positional_args, args.len(), args.collect::<Vec<_>>())) + } + args.map(|arg| arg.into_vec()).collect() +} + +pub fn args_for_exec(current_prog_name: &str, no_of_positional_args: usize) -> (Vec<Vec<u8>>, Vec<Vec<u8>>) { + let mut args = std::env::args_os(); + // remove argv[0] + let _ = args.nth(0); + let mut args = args.map(|arg| arg.into_vec()); + let mut pos_args = vec![]; + // get positional args + for i in 1..no_of_positional_args+1 { + pos_args.push( + args.nth(0).expect( + &format!("{}: expects {} positional args, only got {}", current_prog_name, no_of_positional_args, i)) + ); + } + // prog... is the rest of the iterator + let prog : Vec<Vec<u8>> = args.collect(); + (pos_args, prog) +} + +pub fn exec_into_args<'a, 'b, Args, Arg, Env, Key, Val>(current_prog_name: &str, args: Args, env_additions: Env) -> ! + where + Args: IntoIterator<Item = Arg>, + Arg: AsRef<[u8]>, + Env: IntoIterator<Item = (Key, Val)>, + Key: AsRef<[u8]>, + Val: AsRef<[u8]>, +{ + // TODO: is this possible without collecting into a Vec first, just leaving it an IntoIterator? + let args = args.into_iter().collect::<Vec<Arg>>(); + let mut args = args.iter().map(|v| OsStr::from_bytes(v.as_ref())); + let prog = args.nth(0).expect(&format!("{}: first argument must be an executable", current_prog_name)); + // TODO: same here + let env = env_additions.into_iter().collect::<Vec<(Key, Val)>>(); + let env = env.iter().map(|(k,v)| (OsStr::from_bytes(k.as_ref()), OsStr::from_bytes(v.as_ref()))); + let err = std::process::Command::new(prog).args(args).envs(env).exec(); + die_missing_executable(current_prog_name, format!("exec failed: {}, while trying to execing into {:?}", err, prog)); +} + +/// Exit 1 to signify a generic expected error +/// (e.g. something that sometimes just goes wrong, like a nix build). +pub fn die_expected_error<S>(current_prog_name: &str, msg: S) -> ! +where S: AsRef<str> +{ + die_with(1, current_prog_name, msg) +} + +/// Exit 100 to signify a user error (“the user is holding it wrong”). +/// This is a permanent error, if the program is executed the same way +/// it should crash with 100 again. +pub fn die_user_error<S>(current_prog_name: &str, msg: S) -> ! +where S: AsRef<str> +{ + die_with(100, current_prog_name, msg) +} + +/// Exit 101 to signify an unexpected crash (failing assertion or panic). +/// This is the same exit code that `panic!()` emits. +pub fn die_panic<S>(current_prog_name: &str, msg: S) -> ! +where S: AsRef<str> +{ + die_with(101, current_prog_name, msg) +} + +/// Exit 111 to signify a temporary error (such as resource exhaustion) +pub fn die_temporary<S>(current_prog_name: &str, msg: S) -> ! +where S: AsRef<str> +{ + die_with(111, current_prog_name, msg) +} + +/// Exit 126 to signify an environment problem +/// (the user has set up stuff incorrectly so the program cannot work) +pub fn die_environment_problem<S>(current_prog_name: &str, msg: S) -> ! +where S: AsRef<str> +{ + die_with(126, current_prog_name, msg) +} + +/// Exit 127 to signify a missing executable. +pub fn die_missing_executable<S>(current_prog_name: &str, msg: S) -> ! +where S: AsRef<str> +{ + die_with(127, current_prog_name, msg) +} + +fn die_with<S>(status: i32, current_prog_name: &str, msg: S) -> ! + where S: AsRef<str> +{ + eprintln!("{}: {}", current_prog_name, msg.as_ref()); + std::process::exit(status) +} diff --git a/users/Profpatsch/imap-idle.nix b/users/Profpatsch/imap-idle.nix new file mode 100644 index 000000000000..3ad5375d89af --- /dev/null +++ b/users/Profpatsch/imap-idle.nix @@ -0,0 +1,14 @@ +{ depot, pkgs, lib, ... }: + +let + imap-idle = depot.nix.writers.rustSimple { + name = "imap-idle"; + dependencies = [ + depot.users.Profpatsch.arglib.netencode.rust + depot.third_party.rust-crates.imap + depot.third_party.rust-crates.epoll + depot.users.Profpatsch.execline.exec-helpers + ]; + } (builtins.readFile ./imap-idle.rs); + +in imap-idle diff --git a/users/Profpatsch/imap-idle.rs b/users/Profpatsch/imap-idle.rs new file mode 100644 index 000000000000..9dce736d0d8a --- /dev/null +++ b/users/Profpatsch/imap-idle.rs @@ -0,0 +1,132 @@ +extern crate exec_helpers; +// extern crate arglib_netencode; +// extern crate netencode; +extern crate imap; +extern crate epoll; + +// use netencode::dec; +use std::convert::TryFrom; +use std::io::{Read, Write}; +use std::fs::File; +use std::os::unix::io::{FromRawFd, AsRawFd, RawFd}; +use std::time::Duration; +use imap::extensions::idle::SetReadTimeout; + +/// Implements an UCSPI client that wraps fd 6 & 7 +/// and implements Write and Read with a timeout. +/// See https://cr.yp.to/proto/ucspi.txt +#[derive(Debug)] +struct UcspiClient { + read: File, + read_epoll_fd: RawFd, + read_timeout: Option<Duration>, + write: File, +} + +impl UcspiClient { + /// Use fd 6 and 7 to connect to the net, as is specified. + /// Unsafe because fd 6 and 7 are global resources and we don’t mutex them. + pub unsafe fn new_from_6_and_7() -> std::io::Result<Self> { + unsafe { + let read_epoll_fd = epoll::create(false)?; + Ok(UcspiClient { + read: File::from_raw_fd(6), + read_epoll_fd, + read_timeout: None, + write: File::from_raw_fd(7) + }) + } + } +} + +/// Emulates set_read_timeout() like on a TCP socket with an epoll on read. +/// The BSD socket API is rather bad, so fd != fd, +/// and if we cast the `UcspiClient` fds to `TcpStream` instead of `File`, +/// we’d break any UCSPI client programs that *don’t* connect to TCP. +/// Instead we use the (linux) `epoll` API in read to wait on the timeout. +impl SetReadTimeout for UcspiClient { + fn set_read_timeout(&mut self, timeout: Option<Duration>) -> imap::Result<()> { + self.read_timeout = timeout; + Ok(()) + } +} + +impl Read for UcspiClient { + // TODO: test the epoll code with a short timeout + fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> { + const NO_DATA : u64 = 0; + // in order to implement the read_timeout, + // we use epoll to wait for either data or time out + epoll::ctl( + self.read_epoll_fd, + epoll::ControlOptions::EPOLL_CTL_ADD, + self.read.as_raw_fd(), + epoll::Event::new(epoll::Events::EPOLLIN, NO_DATA) + )?; + let UNUSED = epoll::Event::new(epoll::Events::EPOLLIN, NO_DATA); + let wait = epoll::wait( + self.read_epoll_fd, + match self.read_timeout { + Some(duration) => i32::try_from(duration.as_millis()).expect("duration too big for epoll"), + None => -1 // infinite + }, + // event that was generated; but we don’t care + &mut vec![UNUSED; 1][..], + ); + // Delete the listen fd from the epoll fd before reacting + // (otherwise it fails on the next read with `EPOLL_CTL_ADD`) + epoll::ctl( + self.read_epoll_fd, + epoll::ControlOptions::EPOLL_CTL_DEL, + self.read.as_raw_fd(), + UNUSED + )?; + match wait { + // timeout happened (0 events) + Ok(0) => Err(std::io::Error::new(std::io::ErrorKind::TimedOut, "ucspi read timeout")), + // its ready for reading, we can read + Ok(_) => self.read.read(buf), + // error + err => err, + } + } +} + +/// Just proxy through the `Write` of the write fd. +impl Write for UcspiClient { + fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { + self.write.write(buf) + } + fn flush(&mut self) -> std::io::Result<()> { + self.write.flush() + } +} + +/// Connect to IMAP account and listen for new mails on the INBOX. +fn main() { + exec_helpers::no_args("imap-idle"); + + // TODO: use arglib_netencode + let username = std::env::var("IMAP_USERNAME").expect("username"); + let password = std::env::var("IMAP_PASSWORD").expect("password"); + + let net = unsafe { + UcspiClient::new_from_6_and_7().expect("no ucspi client for you") + }; + let client = imap::Client::new(net); + let mut session = client.login(username, password).map_err(|(err, _)| err).expect("unable to login"); + eprintln!("{:#?}", session); + let list = session.list(None, Some("*")); + eprintln!("{:#?}", list); + let mailbox = session.examine("INBOX"); + eprintln!("{:#?}", mailbox); + fn now() -> String { + String::from_utf8_lossy(&std::process::Command::new("date").output().unwrap().stdout).trim_right().to_string() + } + loop { + eprintln!("{}: idling on INBOX", now()); + let mut handle = session.idle().expect("cannot idle on INBOX"); + let () = handle.wait_keepalive().expect("waiting on idle failed"); + eprintln!("{}: The mailbox has changed!", now()); + } +} diff --git a/users/Profpatsch/lens.nix b/users/Profpatsch/lens.nix new file mode 100644 index 000000000000..58d9c27f5242 --- /dev/null +++ b/users/Profpatsch/lens.nix @@ -0,0 +1,136 @@ +{ ... }: +let + id = x: x; + + const = x: y: x; + + comp = f: g: x: f (g x); + + _ = v: f: f v; + + # Profunctor (p :: Type -> Type -> Type) + Profunctor = rec { + # dimap :: (a -> b) -> (c -> d) -> p b c -> p a d + dimap = f: g: x: lmap f (rmap g x); + # lmap :: (a -> b) -> p b c -> p a c + lmap = f: dimap f id; + # rmap :: (c -> d) -> p b c -> p b d + rmap = g: dimap id g; + }; + + # Profunctor (->) + profunctorFun = Profunctor // { + # dimap :: (a -> b) -> (c -> d) -> (b -> c) -> a -> d + dimap = ab: cd: bc: a: cd (bc (ab a)); + # lmap :: (a -> b) -> (b -> c) -> (a -> c) + lmap = ab: bc: a: bc (ab a); + # rmap :: (c -> d) -> (b -> c) -> (b -> d) + rmap = cd: bc: b: cd (bc b); + }; + + tuple = fst: snd: { + inherit fst snd; + }; + + swap = {fst, snd}: { + fst = snd; + snd = fst; + }; + + # Profunctor p => Strong (p :: Type -> Type -> Type) + Strong = pro: pro // rec { + # firstP :: p a b -> p (a, c) (b, c) + firstP = pab: pro.dimap swap swap (pro.secondP pab); + # secondP :: p a b -> p (c, a) (c, b) + secondP = pab: pro.dimap swap swap (pro.firstP pab); + }; + + # Strong (->) + strongFun = Strong profunctorFun // { + # firstP :: (a -> b) -> (a, c) -> (b, c) + firstP = f: { fst, snd }: { fst = f fst; inherit snd; }; + # secondP :: (a -> b) -> (c, a) -> (c, b) + secondP = f: { snd, fst }: { snd = f snd; inherit fst; }; + }; + + # Iso s t a b :: forall p. Profunctor p -> p a b -> p s t + + # iso :: (s -> a) -> (b -> t) -> Iso s t a b + iso = pro: pro.dimap; + + # Lens s t a b :: forall p. Strong p -> p a b -> p s t + + # lens :: (s -> a) -> (s -> b -> t) -> Lens s t a b + lens = strong: get: set: pab: + lensP + strong + (s: tuple (get s) (b: set s b)) + pab; + + # lensP :: (s -> (a, b -> t)) -> Lens s t a b + lensP = strong: to: pab: + strong.dimap + to + ({fst,snd}: snd fst) + (strong.firstP pab); + + # first element of a tuple + # _1 :: Lens (a, c) (b, c) a b + _1 = strong: strong.firstP; + + # second element of a tuple + # _2 :: Lens (c, a) (c, b) a b + _2 = strong: strong.secondP; + + # a the given field in the record + # field :: (f :: String) -> Lens { f :: a; ... } { f :: b; ... } a b + field = name: strong: + lens + strong + (attrs: attrs.${name}) + (attrs: a: attrs // { ${name} = a; }); + + # Setter :: (->) a b -> (->) s t + # Setter :: (a -> b) -> (s -> t) + + + # Subclasses of profunctor for (->). + # We only have Strong for now, but when we implement Choice we need to add it here. + profunctorSubclassesFun = strongFun; + + # over :: Setter s t a b -> (a -> b) -> s -> t + over = setter: + # A setter needs to be instanced to the profunctor-subclass instances of (->). + (setter profunctorSubclassesFun); + + # set :: Setter s t a b -> b -> s -> t + set = setter: b: over setter (const b); + + # combine a bunch of optics, for the subclass instance of profunctor you give it. + optic = accessors: profunctorSubclass: + builtins.foldl' comp id + (map (accessor: accessor profunctorSubclass) accessors); + + +in { + inherit + id + _ + const + comp + Profunctor + profunctorFun + Strong + strongFun + iso + lens + optic + _1 + _2 + field + tuple + swap + over + set + ; +} diff --git a/users/Profpatsch/lib.nix b/users/Profpatsch/lib.nix new file mode 100644 index 000000000000..e3d59b7d8beb --- /dev/null +++ b/users/Profpatsch/lib.nix @@ -0,0 +1,81 @@ +{ depot, pkgs, ... }: +let + bins = depot.nix.getBins pkgs.coreutils [ "printf" "echo" "cat" "printenv" "tee" ] + // depot.nix.getBins pkgs.bash [ "bash" ] + // depot.nix.getBins pkgs.fdtools [ "multitee" ] + ; + + # Print `msg` and and argv to stderr, then execute into argv + debugExec = msg: depot.nix.writeExecline "debug-exec" {} [ + "if" [ + "fdmove" "-c" "1" "2" + "if" [ bins.printf "%s: " msg ] + "if" [ bins.echo "$@" ] + ] + "$@" + ]; + + # Print stdin to stderr and stdout + eprint-stdin = depot.nix.writeExecline "eprint-stdin" {} [ + "pipeline" [ bins.multitee "0-1,2" ] "$@" + ]; + + # Assume the input on stdin is netencode, pretty print it to stderr and forward it to stdout + eprint-stdin-netencode = depot.nix.writeExecline "eprint-stdin-netencode" {} [ + "pipeline" [ + # move stdout to 3 + "fdmove" "3" "1" + # the multitee copies stdin to 1 (the other pipeline end) and 3 (the stdout of the outer pipeline block) + "pipeline" [ bins.multitee "0-1,3" ] + # make stderr the stdout of pretty, merging with the stderr of pretty + "fdmove" "-c" "1" "2" + depot.users.Profpatsch.netencode.pretty + ] + "$@" + ]; + + # print the given environment variable in $1 to stderr, then execute into the rest of argv + eprintenv = depot.nix.writeExecline "eprintenv" { readNArgs = 1; } [ + "ifelse" [ "fdmove" "-c" "1" "2" bins.printenv "$1" ] + [ "$@" ] + "if" [ depot.tools.eprintf "eprintenv: could not find \"\${1}\" in the environment\n" ] + "$@" + ]; + + # Split stdin into two commands, given by a block and the rest of argv + # + # Example (execline): + # + # pipeline [ echo foo ] + # split-stdin [ fdmove 1 2 foreground [ cat ] echo "bar" ] cat + # + # stdout: foo\n + # stderr: foo\nbar\n + split-stdin = depot.nix.writeExecline "split-stdin" { argMode = "env"; } [ + "pipeline" [ + # this is horrible yes but the quickest way I knew how to implement it + "runblock" "1" bins.bash "-c" ''${bins.tee} >("$@")'' "bash-split-stdin" + ] + "runblock" "-r" "1" + ]; + + # remove everything but a few selected environment variables + runInEmptyEnv = keepVars: + let + importas = pkgs.lib.concatMap (var: [ "importas" "-i" var var ]) keepVars; + # we have to explicitely call export here, because PATH is probably empty + export = pkgs.lib.concatMap (var: [ "${pkgs.execline}/bin/export" var ''''${${var}}'' ]) keepVars; + in depot.nix.writeExecline "empty-env" {} + (importas ++ [ "emptyenv" ] ++ export ++ [ "${pkgs.execline}/bin/exec" "$@" ]); + + +in { + inherit + debugExec + eprint-stdin + eprint-stdin-netencode + eprintenv + split-stdin + runInEmptyEnv + ; +} diff --git a/users/Profpatsch/netencode/README.md b/users/Profpatsch/netencode/README.md new file mode 100644 index 000000000000..67cb843a58c7 --- /dev/null +++ b/users/Profpatsch/netencode/README.md @@ -0,0 +1,111 @@ +# netencode 0.1-unreleased + +[bencode][] and [netstring][]-inspired pipe format that should be trivial to generate correctly in every context (only requires a `byte_length()` and a `printf()`), easy to parse (100 lines of code or less), mostly human-decipherable for easy debugging, and support nested record and sum types. + + +## scalars + +Scalars have the format `[type prefix][size]:[value],`. + +where size is a natural number without leading zeroes. + +### unit + +The unit (`u`) has only one value. + +* The unit is: `u,` + +### numbers + +Naturals (`n`) and Integers (`i`), with a maximum size in bits. + +Bit sizes are specified in 2^n increments, 1 to 9 (`n1`..`n9`, `i1`..`n9`). + +* Natural `1234` that fits in 32 bits (2^5): `n5:1234,` +* Integer `-42` that fits in 8 bits (2^3): `i3:-42,` +* Integer `23` that fits in 64 bits (2^6): `i6:23,` +* Integer `-1` that fits in 512 bits (2^9): `i9:-1,` +* Natural `0` that fits in 1 bit (2^1): `n1:0,` + +An implementation can define the biggest numbers it supports, and has to throw an error for anything bigger. It has to support everything smaller, so for example if you support up to i6/n6, you have to support 1–6 as well. An implementation could support up to the current architecture’s wordsize for example. + +Floats are not supported, you can implement fixed-size decimals or ratios using integers. + +### booleans + +A boolean is represented as `n1`. + +* `n1:0,`: false +* `n1:1,`: true + +TODO: should we add `f,` and `t,`? + +### text + +Text (`t`) that *must* be encoded as UTF-8, starting with its length in bytes: + +* The string `hello world` (11 bytes): `t11:hello world,` +* The string `今日は` (9 bytes): `t9:今日は,` +* The string `:,` (2 bytes): `t2::,,` +* The empty sting `` (0 bytes): `t0:,` + +### binary + +Arbitrary binary strings (`b`) that can contain any data, starting with its length in bytes. + +* The ASCII string `hello world` as binary data (11 bytes): `b11:hello world,` +* The empty binary string (0 bytes): `b0:,` +* The bytestring with `^D` (1 byte): `b1:,` + +Since the binary strings are length-prefixd, they can contain `\0` and no escaping is required. Care has to be taken in languages with `\0`-terminated bytestrings. + +Use text (`t`) if you have utf-8 encoded data. + +## tagged values + +### tags + +A tag (`<`) gives a value a name. The tag is UTF-8 encoded, starting with its length in bytes and proceeding with the value. + +* The tag `foo` (3 bytes) tagging the text `hello` (5 bytes): `<3:foo|t5:hello,` +* The tag `` (0 bytes) tagging the 8-bit integer 0: `<0:|i3:0,` + +### records (products/records), also maps + +A record (`{`) is a concatenation of tags (`<`). It needs to be closed with `}`. +If tag names repeat the later ones should be ignored. Ordering does not matter. + +Similar to text, records start with the length of their *whole encoded content*, in bytes. This makes it possible to treat their contents as opaque bytestrings. + +* There is no empty record. (TODO: make the empty record the unit type, remove `u,`?) +* A record with one empty field, `foo`: `{9:<3:foo|u,}` +* A record with two fields, `foo` and `x`: `{21:<3:foo|u,<1:x|t3:baz,}` +* The same record: `{21:<1:x|t3:baz,<3:foo|u,}` +* The same record (later occurences of fields are ignored): `{28:<1:x|t3:baz,<3:foo|u,<1:x|u,}` + +### sums (tagged unions) + +Simply a tagged value. The tag marker `<` indicates it is a sum if it appears outside of a record. + +## lists + +A list (`[`) imposes an ordering on a sequence of values. It needs to be closed with `]`. Values in it are simply concatenated. + +Similar to records, lists start with the length of their whole encoded content. + +* The empty list: `[0:]` +* The list with one element, the string `foo`: `[7:t3:foo,]` +* The list with text `foo` followed by i3 `-42`: `[14:t3:foo,i3:-42,]` +* The list with `Some` and `None` tags: `[33:<4:Some|t3:foo,<4None|u,<4None|u,]` + +## motivation + +TODO + +## guarantees + +TODO: do I want unique representation (bijection like bencode?) This would put more restrictions on the generator, like sorting records in lexicographic order, but would make it possible to compare without decoding + + +[bencode]: https://en.wikipedia.org/wiki/Bencode +[netstring]: https://en.wikipedia.org/wiki/Netstring diff --git a/users/Profpatsch/netencode/default.nix b/users/Profpatsch/netencode/default.nix new file mode 100644 index 000000000000..739bda3d78c9 --- /dev/null +++ b/users/Profpatsch/netencode/default.nix @@ -0,0 +1,149 @@ +{ depot, pkgs, lib, ... }: + +let + netencode-rs = depot.nix.writers.rustSimpleLib { + name = "netencode"; + dependencies = [ + depot.third_party.rust-crates.nom + depot.users.Profpatsch.execline.exec-helpers + ]; + } (builtins.readFile ./netencode.rs); + + gen = import ./gen.nix { inherit lib; }; + + pretty-rs = depot.nix.writers.rustSimpleLib { + name = "netencode-pretty"; + dependencies = [ + netencode-rs + ]; + } (builtins.readFile ./pretty.rs); + + pretty = depot.nix.writers.rustSimple { + name = "netencode-pretty"; + dependencies = [ + netencode-rs + pretty-rs + depot.users.Profpatsch.execline.exec-helpers + ]; + } '' + extern crate netencode; + extern crate netencode_pretty; + extern crate exec_helpers; + + fn main() { + let (_, prog) = exec_helpers::args_for_exec("netencode-pretty", 0); + let mut buf = vec![]; + let u = netencode::u_from_stdin_or_die_user_error("netencode-pretty", &mut buf); + match netencode_pretty::Pretty::from_u(u).print_multiline(&mut std::io::stdout()) { + Ok(()) => {}, + Err(err) => exec_helpers::die_temporary("netencode-pretty", format!("could not write to stdout: {}", err)) + } + } + ''; + + netencode-mustache = depot.nix.writers.rustSimple { + name = "netencode_mustache"; + dependencies = [ + depot.users.Profpatsch.arglib.netencode.rust + netencode-rs + depot.third_party.rust-crates.mustache + ]; + } (builtins.readFile ./netencode-mustache.rs); + + + record-get = depot.nix.writers.rustSimple { + name = "record-get"; + dependencies = [ + netencode-rs + depot.users.Profpatsch.execline.exec-helpers + depot.users.Profpatsch.arglib.netencode.rust + ]; + } '' + extern crate netencode; + extern crate arglib_netencode; + extern crate exec_helpers; + use netencode::{encode, dec}; + use netencode::dec::{Decoder, DecodeError}; + + fn main() { + let mut buf = vec![]; + let args = exec_helpers::args("record-get", 1); + let field = match std::str::from_utf8(&args[0]) { + Ok(f) => f, + Err(_e) => exec_helpers::die_user_error("record-get", format!("The field name needs to be valid unicode")) + }; + let u = netencode::u_from_stdin_or_die_user_error("record-get", &mut buf); + match (dec::RecordDot {field, inner: dec::AnyU }).dec(u) { + Ok(u) => encode(&mut std::io::stdout(), &u).expect("encoding to stdout failed"), + Err(DecodeError(err)) => exec_helpers::die_user_error("record-get", err) + } + } + ''; + + record-splice-env = depot.nix.writers.rustSimple { + name = "record-splice-env"; + dependencies = [ + netencode-rs + depot.users.Profpatsch.execline.exec-helpers + ]; + } '' + extern crate netencode; + extern crate exec_helpers; + use netencode::dec::{Record, Try, ScalarAsBytes, Decoder, DecodeError}; + + fn main() { + let mut buf = vec![]; + let u = netencode::u_from_stdin_or_die_user_error("record-splice-env", &mut buf); + let (_, prog) = exec_helpers::args_for_exec("record-splice-env", 0); + match Record(Try(ScalarAsBytes)).dec(u) { + Ok(map) => { + exec_helpers::exec_into_args( + "record-splice-env", + prog, + // some elements can’t be decoded as scalars, so just ignore them + map.into_iter().filter_map(|(k, v)| v.map(|v2| (k, v2))) + ); + }, + Err(DecodeError(err)) => exec_helpers::die_user_error("record-splice-env", err), + } + } + ''; + + env-splice-record = depot.nix.writers.rustSimple { + name = "env-splice-record"; + dependencies = [ + netencode-rs + depot.users.Profpatsch.execline.exec-helpers + ]; + } '' + extern crate netencode; + extern crate exec_helpers; + use netencode::{T}; + use std::os::unix::ffi::OsStringExt; + + fn main() { + exec_helpers::no_args("env-splice-record"); + let mut res = std::collections::HashMap::new(); + for (key, val) in std::env::vars_os() { + match (String::from_utf8(key.into_vec()), String::from_utf8(val.into_vec())) { + (Ok(k), Ok(v)) => { let _ = res.insert(k, T::Text(v)); }, + // same as in record-splice-env, we ignore non-utf8 variables + (_, _) => {}, + } + } + netencode::encode(&mut std::io::stdout(), &T::Record(res).to_u()).unwrap() + } + ''; + +in depot.nix.readTree.drvTargets { + inherit + netencode-rs + pretty-rs + pretty + netencode-mustache + record-get + record-splice-env + env-splice-record + gen + ; +} diff --git a/users/Profpatsch/netencode/gen.nix b/users/Profpatsch/netencode/gen.nix new file mode 100644 index 000000000000..305ff7b08dd6 --- /dev/null +++ b/users/Profpatsch/netencode/gen.nix @@ -0,0 +1,69 @@ +{ lib }: +let + + netstring = tag: suffix: s: + "${tag}${toString (builtins.stringLength s)}:${s}${suffix}"; + + unit = "u,"; + + n1 = b: if b then "n1:1," else "n1:0,"; + + n = i: n: "n${toString i}:${toString n},"; + i = i: n: "i${toString i}:${toString n},"; + + n3 = n 3; + n6 = n 6; + n7 = n 7; + + i3 = i 3; + i6 = i 6; + i7 = i 7; + + text = netstring "t" ","; + binary = netstring "b" ","; + + tag = key: val: netstring "<" "|" key + val; + + concatStrings = builtins.concatStringsSep ""; + + record = lokv: netstring "{" "}" + (concatStrings (map ({key, val}: tag key val) lokv)); + + list = l: netstring "[" "]" (concatStrings l); + + dwim = val: + let match = { + "bool" = n1; + "int" = i6; + "string" = text; + "set" = attrs: + # it could be a derivation, then just return the path + if attrs.type or "" == "derivation" then text "${attrs}" + else + record (lib.mapAttrsToList + (k: v: { + key = k; + val = dwim v; + }) attrs); + "list" = l: list (map dwim l); + }; + in match.${builtins.typeOf val} val; + +in { + inherit + unit + n1 + n3 + n6 + n7 + i3 + i6 + i7 + text + binary + tag + record + list + dwim + ; +} diff --git a/users/Profpatsch/netencode/netencode-mustache.rs b/users/Profpatsch/netencode/netencode-mustache.rs new file mode 100644 index 000000000000..ee7bafed2250 --- /dev/null +++ b/users/Profpatsch/netencode/netencode-mustache.rs @@ -0,0 +1,53 @@ +extern crate netencode; +extern crate mustache; +extern crate arglib_netencode; + +use mustache::{Data}; +use netencode::{T}; +use std::collections::HashMap; +use std::os::unix::ffi::{OsStrExt}; +use std::io::{Read}; + +fn netencode_to_mustache_data_dwim(t: T) -> Data { + match t { + // TODO: good idea? + T::Unit => Data::Null, + T::N1(b) => Data::Bool(b), + T::N3(u) => Data::String(u.to_string()), + T::N6(u) => Data::String(u.to_string()), + T::N7(u) => Data::String(u.to_string()), + T::I3(i) => Data::String(i.to_string()), + T::I6(i) => Data::String(i.to_string()), + T::I7(i) => Data::String(i.to_string()), + T::Text(s) => Data::String(s), + T::Binary(b) => unimplemented!(), + T::Sum(tag) => unimplemented!(), + T::Record(xs) => Data::Map( + xs.into_iter() + .map(|(key, val)| (key, netencode_to_mustache_data_dwim(val))) + .collect::<HashMap<_,_>>() + ), + T::List(xs) => Data::Vec( + xs.into_iter() + .map(|x| netencode_to_mustache_data_dwim(x)) + .collect::<Vec<_>>() + ), + } +} + +pub fn from_stdin() -> () { + let data = netencode_to_mustache_data_dwim( + arglib_netencode::arglib_netencode("netencode-mustache", Some(std::ffi::OsStr::new("TEMPLATE_DATA"))) + ); + let mut stdin = String::new(); + std::io::stdin().read_to_string(&mut stdin).unwrap(); + mustache::compile_str(&stdin) + .and_then(|templ| templ.render_data( + &mut std::io::stdout(), + &data + )).unwrap() +} + +pub fn main() { + from_stdin() +} diff --git a/users/Profpatsch/netencode/netencode.rs b/users/Profpatsch/netencode/netencode.rs new file mode 100644 index 000000000000..5bd43f992f0b --- /dev/null +++ b/users/Profpatsch/netencode/netencode.rs @@ -0,0 +1,844 @@ +extern crate nom; +extern crate exec_helpers; + +use std::collections::HashMap; +use std::io::{Write, Read}; +use std::fmt::{Display, Debug}; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum T { + // Unit + Unit, + // Boolean + N1(bool), + // Naturals + N3(u8), + N6(u64), + N7(u128), + // Integers + I3(i8), + I6(i64), + I7(i128), + // Text + // TODO: make into &str + Text(String), + // TODO: rename to Bytes + Binary(Vec<u8>), + // Tags + // TODO: make into &str + // TODO: rename to Tag + Sum(Tag<String, T>), + // TODO: make into &str + Record(HashMap<String, T>), + List(Vec<T>), +} + +impl T { + pub fn to_u<'a>(&'a self) -> U<'a> { + match self { + T::Unit => U::Unit, + T::N1(b) => U::N1(*b), + T::N3(u) => U::N3(*u), + T::N6(u) => U::N6(*u), + T::N7(u) => U::N7(*u), + T::I3(i) => U::I3(*i), + T::I6(i) => U::I6(*i), + T::I7(i) => U::I7(*i), + T::Text(t) => U::Text(t.as_str()), + T::Binary(v) => U::Binary(v), + T::Sum(Tag { tag, val }) => U::Sum( + Tag { tag: tag.as_str(), val: Box::new(val.to_u()) } + ), + T::Record(map) => U::Record( + map.iter().map(|(k, v)| (k.as_str(), v.to_u())).collect() + ), + T::List(l) => U::List( + l.iter().map(|v| v.to_u()).collect::<Vec<U<'a>>>() + ), + } + } + + pub fn encode<'a>(&'a self) -> Vec<u8> { + match self { + // TODO: don’t go via U, inefficient + o => o.to_u().encode() + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum U<'a> { + Unit, + // Boolean + N1(bool), + // Naturals + N3(u8), + N6(u64), + N7(u128), + // Integers + I3(i8), + I6(i64), + I7(i128), + // Text + Text(&'a str), + Binary(&'a [u8]), + // TODO: the U-recursion we do here means we can’t be breadth-lazy anymore + // like we originally planned; maybe we want to go `U<'a>` → `&'a [u8]` again? + // Tags + // TODO: rename to Tag + Sum(Tag<&'a str, U<'a>>), + Record(HashMap<&'a str, U<'a>>), + List(Vec<U<'a>>), +} + +impl<'a> U<'a> { + pub fn encode(&self) -> Vec<u8> { + let mut c = std::io::Cursor::new(vec![]); + encode(&mut c, self); + c.into_inner() + } + + pub fn to_t(&self) -> T { + match self { + U::Unit => T::Unit, + U::N1(b) => T::N1(*b), + U::N3(u) => T::N3(*u), + U::N6(u) => T::N6(*u), + U::N7(u) => T::N7(*u), + U::I3(i) => T::I3(*i), + U::I6(i) => T::I6(*i), + U::I7(i) => T::I7(*i), + U::Text(t) => T::Text((*t).to_owned()), + U::Binary(v) => T::Binary((*v).to_owned()), + U::Sum(Tag { tag, val }) => T::Sum( + Tag { tag: (*tag).to_owned(), val: Box::new(val.to_t()) } + ), + U::Record(map) => T::Record( + map.iter().map(|(k, v)| ((*k).to_owned(), v.to_t())).collect::<HashMap<String, T>>() + ), + U::List(l) => T::List( + l.iter().map(|v| v.to_t()).collect::<Vec<T>>() + ), + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Tag<S, A> { + // TODO: make into &str + pub tag: S, + pub val: Box<A> +} + +impl<S, A> Tag<S, A> { + fn map<F, B>(self, f: F) -> Tag<S, B> + where F: Fn(A) -> B { + Tag { + tag: self.tag, + val: Box::new(f(*self.val)) + } + } +} + +fn encode_tag<W: Write>(w: &mut W, tag: &str, val: &U) -> std::io::Result<()> { + write!(w, "<{}:{}|", tag.len(), tag)?; + encode(w, val)?; + Ok(()) +} + +pub fn encode<W: Write>(w: &mut W, u: &U) -> std::io::Result<()> { + match u { + U::Unit => write!(w, "u,"), + U::N1(b) => if *b { write!(w, "n1:1,") } else { write!(w, "n1:0,") }, + U::N3(n) => write!(w, "n3:{},", n), + U::N6(n) => write!(w, "n6:{},", n), + U::N7(n) => write!(w, "n7:{},", n), + U::I3(i) => write!(w, "i3:{},", i), + U::I6(i) => write!(w, "i6:{},", i), + U::I7(i) => write!(w, "i7:{},", i), + U::Text(s) => { + write!(w, "t{}:", s.len()); + w.write_all(s.as_bytes()); + write!(w, ",") + } + U::Binary(s) => { + write!(w, "b{}:", s.len()); + w.write_all(&s); + write!(w, ",") + }, + U::Sum(Tag{tag, val}) => encode_tag(w, tag, val), + U::Record(m) => { + let mut c = std::io::Cursor::new(vec![]); + for (k, v) in m { + encode_tag(&mut c, k, v)?; + } + write!(w, "{{{}:", c.get_ref().len())?; + w.write_all(c.get_ref())?; + write!(w, "}}") + }, + U::List(l) => { + let mut c = std::io::Cursor::new(vec![]); + for u in l { + encode(&mut c, u)?; + } + write!(w, "[{}:", c.get_ref().len())?; + w.write_all(c.get_ref())?; + write!(w, "]") + } + } +} + +pub fn text(s: String) -> T { + T::Text(s) +} + +pub fn u_from_stdin_or_die_user_error<'a>(prog_name: &'_ str, stdin_buf: &'a mut Vec<u8>) -> U<'a> { + std::io::stdin().lock().read_to_end(stdin_buf); + let u = match parse::u_u(stdin_buf) { + Ok((rest, u)) => match rest { + b"" => u, + _ => exec_helpers::die_user_error(prog_name, format!("stdin contained some soup after netencode value: {:?}", String::from_utf8_lossy(rest))) + }, + Err(err) => exec_helpers::die_user_error(prog_name, format!("unable to parse netencode from stdin: {:?}", err)) + }; + u +} + +pub mod parse { + use super::{T, Tag, U}; + + use std::str::FromStr; + use std::ops::Neg; + use std::collections::HashMap; + + use nom::{IResult}; + use nom::branch::{alt}; + use nom::bytes::streaming::{tag, take}; + use nom::character::streaming::{digit1, char}; + use nom::sequence::{tuple}; + use nom::combinator::{map, map_res, flat_map, map_parser, opt}; + use nom::error::{context, ErrorKind, ParseError}; + + fn unit_t(s: &[u8]) -> IResult<&[u8], ()> { + let (s, _) = context("unit", tag("u,"))(s)?; + Ok((s, ())) + } + + fn usize_t(s: &[u8]) -> IResult<&[u8], usize> { + context( + "usize", + map_res( + map_res(digit1, |n| std::str::from_utf8(n)), + |s| s.parse::<usize>()) + )(s) + } + + fn sized(begin: char, end: char) -> impl Fn(&[u8]) -> IResult<&[u8], &[u8]> { + move |s: &[u8]| { + // This is the point where we check the descriminator; + // if the beginning char does not match, we can immediately return. + let (s, _) = char(begin)(s)?; + let (s, (len, _)) = tuple(( + usize_t, + char(':') + ))(s)?; + let (s, (res, _)) = tuple(( + take(len), + char(end) + ))(s)?; + Ok((s, res)) + } + } + + + fn uint_t<'a, I: FromStr + 'a>(t: &'static str) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], I> { + move |s: &'a [u8]| { + let (s, (_, _, int, _)) = tuple(( + tag(t.as_bytes()), + char(':'), + map_res( + map_res(digit1, |n: &[u8]| std::str::from_utf8(n)), + |s| s.parse::<I>() + ), + char(',') + ))(s)?; + Ok((s, int)) + } + } + + fn bool_t<'a>() -> impl Fn(&'a [u8]) -> IResult<&'a [u8], bool> { + context("bool", alt(( + map(tag("n1:0,"), |_| false), + map(tag("n1:1,"), |_| true), + ))) + } + + fn int_t<'a, I: FromStr + Neg<Output=I>>(t: &'static str) -> impl Fn(&'a [u8]) -> IResult<&[u8], I> { + context( + t, + move |s: &'a [u8]| { + let (s, (_, _, neg, int, _)) = tuple(( + tag(t.as_bytes()), + char(':'), + opt(char('-')), + map_res( + map_res(digit1, |n: &[u8]| std::str::from_utf8(n)), + |s| s.parse::<I>() + ), + char(',') + ))(s)?; + let res = match neg { + Some(_) => -int, + None => int, + }; + Ok((s, res)) + } + ) + } + + fn tag_t(s: &[u8]) -> IResult<&[u8], Tag<String, T>> { + // recurses into the main parser + map(tag_g(t_t), + |Tag {tag, val}| + Tag { + tag: tag.to_string(), + val + })(s) + } + + fn tag_g<'a, P, O>(inner: P) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], Tag<&'a str, O>> + where + P: Fn(&'a [u8]) -> IResult<&'a [u8], O> + { + move |s: &[u8]| { + let (s, tag) = sized('<', '|')(s)?; + let (s, val) = inner(s)?; + Ok((s, Tag { + tag: std::str::from_utf8(tag) + .map_err(|_| nom::Err::Failure((s, ErrorKind::Char)))?, + val: Box::new(val) + })) + + } + } + + /// parse text scalar (`t5:hello,`) + fn text(s: &[u8]) -> IResult<&[u8], T> { + let (s, res) = text_g(s)?; + Ok((s, T::Text(res.to_string()))) + } + + fn text_g(s: &[u8]) -> IResult<&[u8], &str> { + let (s, res) = sized('t', ',')(s)?; + Ok((s, + std::str::from_utf8(res) + .map_err(|_| nom::Err::Failure((s, ErrorKind::Char)))?, + )) + } + + fn binary<'a>() -> impl Fn(&'a [u8]) -> IResult<&'a [u8], T> { + map(binary_g(), |b| T::Binary(b.to_owned())) + } + + fn binary_g() -> impl Fn(&[u8]) -> IResult<&[u8], &[u8]> { + sized('b', ',') + } + + fn list_t(s: &[u8]) -> IResult<&[u8], Vec<T>> { + list_g(t_t)(s) + } + + /// Wrap the inner parser of an `many0`/`fold_many0`, so that the parser + /// is not called when the `s` is empty already, preventing it from + /// returning `Incomplete` on streaming parsing. + fn inner_no_empty_string<'a, P, O>(inner: P) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], O> + where + O: Clone, + P: Fn(&'a [u8]) -> IResult<&'a [u8], O>, + { + move |s: &'a [u8]| { + if s.is_empty() { + // This is a bit hacky, `many0` considers the inside done + // when a parser returns `Err::Error`, ignoring the actual error content + Err(nom::Err::Error((s, nom::error::ErrorKind::Many0))) + } else { + inner(s) + } + } + } + + fn list_g<'a, P, O>(inner: P) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], Vec<O>> + where + O: Clone, + P: Fn(&'a [u8]) -> IResult<&'a [u8], O>, + { + map_parser( + sized('[', ']'), + nom::multi::many0(inner_no_empty_string(inner)) + ) + } + + fn record_t<'a>(s: &'a [u8]) -> IResult<&'a [u8], HashMap<String, T>> { + let (s, r) = record_g(t_t)(s)?; + Ok((s, + r.into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect::<HashMap<_,_>>())) + } + + fn record_g<'a, P, O>(inner: P) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], HashMap<&'a str, O>> + where + O: Clone, + P: Fn(&'a [u8]) -> IResult<&'a [u8], O> + { + move |s: &'a [u8]| { + let (s, map) = map_parser( + sized('{', '}'), + nom::multi::fold_many0( + inner_no_empty_string(tag_g(&inner)), + HashMap::new(), + |mut acc: HashMap<_,_>, Tag { tag, mut val }| { + // ignore duplicated tag names that appear later + // according to netencode spec + if ! acc.contains_key(tag) { + acc.insert(tag, *val); + } + acc + } + ) + )(s)?; + if map.is_empty() { + // records must not be empty, according to the spec + Err(nom::Err::Failure((s,nom::error::ErrorKind::Many1))) + } else { + Ok((s, map)) + } + } + } + + pub fn u_u(s: &[u8]) -> IResult<&[u8], U> { + alt(( + map(text_g, U::Text), + map(binary_g(), U::Binary), + map(unit_t, |()| U::Unit), + map(tag_g(u_u), |t| U::Sum(t)), + map(list_g(u_u), U::List), + map(record_g(u_u), U::Record), + + map(bool_t(), |u| U::N1(u)), + map(uint_t("n3"), |u| U::N3(u)), + map(uint_t("n6"), |u| U::N6(u)), + map(uint_t("n7"), |u| U::N7(u)), + map(int_t("i3"), |u| U::I3(u)), + map(int_t("i6"), |u| U::I6(u)), + map(int_t("i7"), |u| U::I7(u)), + + // less common + map(uint_t("n2"), |u| U::N3(u)), + map(uint_t("n4"), |u| U::N6(u)), + map(uint_t("n5"), |u| U::N6(u)), + map(int_t("i1"), |u| U::I3(u)), + map(int_t("i2"), |u| U::I3(u)), + map(int_t("i4"), |u| U::I6(u)), + map(int_t("i5"), |u| U::I6(u)), + // TODO: 8, 9 not supported + ))(s) + } + + pub fn t_t(s: &[u8]) -> IResult<&[u8], T> { + alt(( + text, + binary(), + map(unit_t, |_| T::Unit), + map(tag_t, |t| T::Sum(t)), + map(list_t, |l| T::List(l)), + map(record_t, |p| T::Record(p)), + + map(bool_t(), |u| T::N1(u)), + // 8, 64 and 128 bit + map(uint_t("n3"), |u| T::N3(u)), + map(uint_t("n6"), |u| T::N6(u)), + map(uint_t("n7"), |u| T::N7(u)), + map(int_t("i3"), |u| T::I3(u)), + map(int_t("i6"), |u| T::I6(u)), + map(int_t("i7"), |u| T::I7(u)), + + // less common + map(uint_t("n2"), |u| T::N3(u)), + map(uint_t("n4"), |u| T::N6(u)), + map(uint_t("n5"), |u| T::N6(u)), + map(int_t("i1"), |u| T::I3(u)), + map(int_t("i2"), |u| T::I3(u)), + map(int_t("i4"), |u| T::I6(u)), + map(int_t("i5"), |u| T::I6(u)), + // TODO: 8, 9 not supported + ))(s) + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn test_parse_unit_t() { + assert_eq!( + unit_t("u,".as_bytes()), + Ok(("".as_bytes(), ())) + ); + } + + #[test] + fn test_parse_bool_t() { + assert_eq!( + bool_t()("n1:0,".as_bytes()), + Ok(("".as_bytes(), false)) + ); + assert_eq!( + bool_t()("n1:1,".as_bytes()), + Ok(("".as_bytes(), true)) + ); + } + + #[test] + fn test_parse_usize_t() { + assert_eq!( + usize_t("32foo".as_bytes()), + Ok(("foo".as_bytes(), 32)) + ); + } + + #[test] + fn test_parse_int_t() { + assert_eq!( + uint_t::<u8>("n3")("n3:42,abc".as_bytes()), + Ok(("abc".as_bytes(), 42)) + ); + assert_eq!( + uint_t::<u8>("n3")("n3:1024,abc".as_bytes()), + Err(nom::Err::Error(("1024,abc".as_bytes(), nom::error::ErrorKind::MapRes))) + ); + assert_eq!( + int_t::<i64>("i6")("i6:-23,abc".as_bytes()), + Ok(("abc".as_bytes(), -23)) + ); + assert_eq!( + int_t::<i128>("i3")("i3:0,:abc".as_bytes()), + Ok((":abc".as_bytes(), 0)) + ); + assert_eq!( + uint_t::<u8>("n7")("n7:09,".as_bytes()), + Ok(("".as_bytes(), 9)) + ); + // assert_eq!( + // length("c"), + // Err(nom::Err::Error(("c", nom::error::ErrorKind::Digit))) + // ); + // assert_eq!( + // length(":"), + // Err(nom::Err::Error((":", nom::error::ErrorKind::Digit))) + // ); + } + + #[test] + fn test_parse_text() { + assert_eq!( + text("t5:hello,".as_bytes()), + Ok(("".as_bytes(), T::Text("hello".to_owned()))), + "{}", r"t5:hello," + ); + assert_eq!( + text("t4:fo".as_bytes()), + // The content of the text should be 4 long + Err(nom::Err::Incomplete(nom::Needed::Size(4))), + "{}", r"t4:fo," + ); + assert_eq!( + text("t9:今日は,".as_bytes()), + Ok(("".as_bytes(), T::Text("今日は".to_owned()))), + "{}", r"t9:今日は," + ); + } + + #[test] + fn test_parse_binary() { + assert_eq!( + binary()("b5:hello,".as_bytes()), + Ok(("".as_bytes(), T::Binary(Vec::from("hello".to_owned())))), + "{}", r"b5:hello," + ); + assert_eq!( + binary()("b4:fo".as_bytes()), + // The content of the byte should be 4 long + Err(nom::Err::Incomplete(nom::Needed::Size(4))), + "{}", r"b4:fo," + ); + assert_eq!( + binary()("b4:foob".as_bytes()), + // The content is 4 bytes now, but the finishing , is missing + Err(nom::Err::Incomplete(nom::Needed::Size(1))), + "{}", r"b4:fo," + ); + assert_eq!( + binary()("b9:今日は,".as_bytes()), + Ok(("".as_bytes(), T::Binary(Vec::from("今日は".as_bytes())))), + "{}", r"b9:今日は," + ); + } + + #[test] + fn test_list() { + assert_eq!( + list_t("[0:]".as_bytes()), + Ok(("".as_bytes(), vec![])), + "{}", r"[0:]" + ); + assert_eq!( + list_t("[6:u,u,u,]".as_bytes()), + Ok(("".as_bytes(), vec![ + T::Unit, + T::Unit, + T::Unit, + ])), + "{}", r"[6:u,u,u,]" + ); + assert_eq!( + list_t("[15:u,[7:t3:foo,]u,]".as_bytes()), + Ok(("".as_bytes(), vec![ + T::Unit, + T::List(vec![T::Text("foo".to_owned())]), + T::Unit, + ])), + "{}", r"[15:u,[7:t3:foo,]u,]" + ); + } + + #[test] + fn test_record() { + assert_eq!( + record_t("{21:<1:a|u,<1:b|u,<1:c|u,}".as_bytes()), + Ok(("".as_bytes(), vec![ + ("a".to_owned(), T::Unit), + ("b".to_owned(), T::Unit), + ("c".to_owned(), T::Unit), + ].into_iter().collect::<HashMap<String, T>>())), + "{}", r"{21:<1:a|u,<1:b|u,<1:c|u,}" + ); + // duplicated keys are ignored (first is taken) + assert_eq!( + record_t("{25:<1:a|u,<1:b|u,<1:a|i1:-1,}".as_bytes()), + Ok(("".as_bytes(), vec![ + ("a".to_owned(), T::Unit), + ("b".to_owned(), T::Unit), + ].into_iter().collect::<HashMap<_,_>>())), + "{}", r"{25:<1:a|u,<1:b|u,<1:a|i1:-1,}" + ); + // empty records are not allowed + assert_eq!( + record_t("{0:}".as_bytes()), + Err(nom::Err::Failure(("".as_bytes(), nom::error::ErrorKind::Many1))), + "{}", r"{0:}" + ); + } + + #[test] + fn test_parse() { + assert_eq!( + t_t("n3:255,".as_bytes()), + Ok(("".as_bytes(), T::N3(255))), + "{}", r"n3:255," + ); + assert_eq!( + t_t("t6:halloo,".as_bytes()), + Ok(("".as_bytes(), T::Text("halloo".to_owned()))), + "{}", r"t6:halloo," + ); + assert_eq!( + t_t("<3:foo|t6:halloo,".as_bytes()), + Ok(("".as_bytes(), T::Sum (Tag { + tag: "foo".to_owned(), + val: Box::new(T::Text("halloo".to_owned())) + }))), + "{}", r"<3:foo|t6:halloo," + ); + // { a: Unit + // , foo: List <A: Unit | B: List i3> } + assert_eq!( + t_t("{52:<1:a|u,<3:foo|[33:<1:A|u,<1:A|n1:1,<1:B|[7:i3:127,]]}".as_bytes()), + Ok(("".as_bytes(), T::Record(vec![ + ("a".to_owned(), T::Unit), + ("foo".to_owned(), T::List(vec![ + T::Sum(Tag { tag: "A".to_owned(), val: Box::new(T::Unit) }), + T::Sum(Tag { tag: "A".to_owned(), val: Box::new(T::N1(true)) }), + T::Sum(Tag { tag: "B".to_owned(), val: Box::new(T::List(vec![T::I3(127)])) }), + ])) + ].into_iter().collect::<HashMap<String, T>>()))), + "{}", r"{52:<1:a|u,<3:foo|[33:<1:A|u,<1:A|n1:1,<1:B|[7:i3:127,]]}" + ); + } + + } +} + +pub mod dec { + use super::*; + use std::collections::HashMap; + + pub struct DecodeError(pub String); + + pub trait Decoder<'a> { + type A; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError>; + } + + /// Any netencode, as `T`. + #[derive(Clone, Copy)] + pub struct AnyT; + /// Any netencode, as `U`. + #[derive(Clone, Copy)] + pub struct AnyU; + + impl<'a> Decoder<'a> for AnyT { + type A = T; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + Ok(u.to_t()) + } + } + + impl<'a> Decoder<'a> for AnyU { + type A = U<'a>; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + Ok(u) + } + } + + /// A text + #[derive(Clone, Copy)] + pub struct Text; + + /// A bytestring + // TODO: rename to Bytes + #[derive(Clone, Copy)] + pub struct Binary; + + impl<'a> Decoder<'a> for Text { + type A = &'a str; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + match u { + U::Text(t) => Ok(t), + other => Err(DecodeError(format!("Cannot decode {:?} into Text", other))), + } + } + } + + impl<'a> Decoder<'a> for Binary { + type A = &'a [u8]; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + match u { + U::Binary(b) => Ok(b), + other => Err(DecodeError(format!("Cannot decode {:?} into Binary", other))), + } + } + } + + /// Any scalar, converted to bytes. + #[derive(Clone, Copy)] + pub struct ScalarAsBytes; + + impl<'a> Decoder<'a> for ScalarAsBytes { + type A = Vec<u8>; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + match u { + U::N3(u) => Ok(format!("{}", u).into_bytes()), + U::N6(u) => Ok(format!("{}", u).into_bytes()), + U::N7(u) => Ok(format!("{}", u).into_bytes()), + U::I3(i) => Ok(format!("{}", i).into_bytes()), + U::I6(i) => Ok(format!("{}", i).into_bytes()), + U::I7(i) => Ok(format!("{}", i).into_bytes()), + U::Text(t) => Ok(t.as_bytes().to_owned()), + U::Binary(b) => Ok(b.to_owned()), + o => Err(DecodeError(format!("Cannot decode {:?} into scalar", o))), + } + } + } + + /// A map of Ts (TODO: rename to map) + #[derive(Clone, Copy)] + pub struct Record<T>(pub T); + + impl<'a, Inner> Decoder<'a> for Record<Inner> + where Inner: Decoder<'a> + { + type A = HashMap<&'a str, Inner::A>; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + match u { + U::Record(map) => + map.into_iter() + .map(|(k, v)| self.0.dec(v).map(|v2| (k, v2))) + .collect::<Result<Self::A, _>>(), + o => Err(DecodeError(format!("Cannot decode {:?} into record", o))) + } + } + } + + /// Assume a record and project out the field with the given name and type. + #[derive(Clone, Copy)] + pub struct RecordDot<'a, T> { + pub field: &'a str, + pub inner: T + } + + impl <'a, Inner> Decoder<'a> for RecordDot<'_, Inner> + where Inner: Decoder<'a> + Clone + { + type A = Inner::A; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + match Record(self.inner.clone()).dec(u) { + Ok(mut map) => match map.remove(self.field) { + Some(inner) => Ok(inner), + None => Err(DecodeError(format!("Cannot find `{}` in record map", self.field))), + }, + Err(err) => Err(err), + } + } + } + + /// Equals one of the listed `A`s exactly, after decoding. + #[derive(Clone)] + pub struct OneOf<T, A>{ + pub inner: T, + pub list: Vec<A>, + } + + impl <'a, Inner> Decoder<'a> for OneOf<Inner, Inner::A> + where Inner: Decoder<'a>, + Inner::A: Display + Debug + PartialEq + { + type A = Inner::A; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + match self.inner.dec(u) { + Ok(inner) => match self.list.iter().any(|x| x.eq(&inner)) { + true => Ok(inner), + false => Err(DecodeError(format!("{} is not one of {:?}", inner, self.list))) + }, + Err(err) => Err(err) + } + } + } + + /// Try decoding as `T`. + #[derive(Clone)] + pub struct Try<T>(pub T); + + impl <'a, Inner> Decoder<'a> for Try<Inner> + where Inner: Decoder<'a> + { + type A = Option<Inner::A>; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + match self.0.dec(u) { + Ok(inner) => Ok(Some(inner)), + Err(err) => Ok(None) + } + } + } + +} diff --git a/users/Profpatsch/netencode/pretty.rs b/users/Profpatsch/netencode/pretty.rs new file mode 100644 index 000000000000..8fec24a60e1a --- /dev/null +++ b/users/Profpatsch/netencode/pretty.rs @@ -0,0 +1,140 @@ +extern crate netencode; + +use netencode::{U, T, Tag}; + +pub enum Pretty { + Single { + r#type: char, + length: String, + val: String, + trailer: char, + }, + Tag { + r#type: char, + length: String, + key: String, + inner: char, + val: Box<Pretty>, + }, + Multi { + r#type: char, + length: String, + vals: Vec<Pretty>, + trailer: char + }, +} + +impl Pretty { + pub fn from_u<'a>(u: U<'a>) -> Pretty { + match u { + U::Unit => Self::scalar('u', "", ""), + U::N1(b) => Self::scalar('n', "1:", if b { "1" } else { "0" }), + U::N3(n) => Self::scalar('n', "3:", n), + U::N6(n) => Self::scalar('n', "6:", n), + U::N7(n) => Self::scalar('n', "7:", n), + U::I3(i) => Self::scalar('i', "3:", i), + U::I6(i) => Self::scalar('i', "6:", i), + U::I7(i) => Self::scalar('i', "7:", i), + U::Text(s) => Pretty::Single { + r#type: 't', + length: format!("{}:", s.len()), + val: s.to_string(), + trailer: ',' + }, + U::Binary(s) => Pretty::Single { + r#type: 'b', + length: format!("{}:", s.len()), + // For pretty printing we want the string to be visible obviously. + // Instead of not supporting binary, let’s use lossy conversion. + val: String::from_utf8_lossy(s).into_owned(), + trailer: ',' + }, + U::Sum(Tag{tag, val}) => Self::pretty_tag(tag, Self::from_u(*val)), + U::Record(m) => Pretty::Multi { + r#type: '{', + // TODO: we are losing the size here, should we recompute it? Keep it? + length: String::from(""), + vals: m.into_iter().map(|(k, v)| Self::pretty_tag(k, Self::from_u(v))).collect(), + trailer: '}' + }, + U::List(l) => Pretty::Multi { + r#type: '[', + // TODO: we are losing the size here, should we recompute it? Keep it? + length: String::from(""), + vals: l.into_iter().map(|v| Self::from_u(v)).collect(), + trailer: ']', + }, + } + } + + fn scalar<D>(r#type: char, length: &str, d: D) -> Pretty + where D: std::fmt::Display + { + Pretty::Single { + r#type, + length: length.to_string(), + val: format!("{}", d), + trailer: ',' + } + } + + fn pretty_tag(tag: &str, val: Pretty) -> Pretty { + Pretty::Tag { + r#type: '<', + length: format!("{}:", tag.len()), + key: tag.to_string(), + inner: '|', + val: Box::new(val), + } + } + + pub fn print_multiline<W>(&self, mut w: &mut W) -> std::io::Result<()> + where W: std::io::Write + { + Self::go(&mut w, self, 0, true); + write!(w, "\n") + } + + fn go<W>(mut w: &mut W, p: &Pretty, depth: usize, is_newline: bool) -> std::io::Result<()> + where W: std::io::Write + { + const full : usize = 4; + const half : usize = 2; + let i = &vec![b' '; depth*full]; + let iandhalf = &vec![b' '; depth*full + half]; + let (i, iandhalf) = unsafe {( + std::str::from_utf8_unchecked(i), + std::str::from_utf8_unchecked(iandhalf), + )}; + if is_newline { + write!(&mut w, "{}", i); + } + match p { + Pretty::Single {r#type, length, val, trailer} => + write!(&mut w, "{} {}{}", r#type, val, trailer), + Pretty::Tag { r#type, length, key, inner, val } => { + write!(&mut w, "{} {} {}", r#type, key, inner)?; + Self::go::<W>(&mut w, val, depth, false) + }, + // if the length is 0 or 1, we print on one line, + // only if there’s more than one element we split the resulting value. + // we never break lines on arbitrary column sizes, since that is just silly. + Pretty::Multi {r#type, length, vals, trailer} => match vals.len() { + 0 => write!(&mut w, "{} {}", r#type, trailer), + 1 => { + write!(&mut w, "{} ", r#type); + Self::go::<W>(&mut w, &vals[0], depth, false)?; + write!(&mut w, "{}", trailer) + }, + more => { + write!(&mut w, "\n{}{} \n", iandhalf, r#type)?; + for v in vals { + Self::go::<W>(&mut w, v, depth + 1, true)?; + write!(&mut w, "\n")?; + } + write!(&mut w, "{}{}", iandhalf, trailer) + } + }, + } + } +} diff --git a/users/Profpatsch/netstring/README.md b/users/Profpatsch/netstring/README.md new file mode 100644 index 000000000000..b8daea11d158 --- /dev/null +++ b/users/Profpatsch/netstring/README.md @@ -0,0 +1,18 @@ +# Netstring + +Netstrings are a djb invention. They are intended as a serialization format. Instead of inline control characters like `\n` or `\0` to signal the end of a string, they use a run-length encoding given as the number of bytes, encoded in ASCII, at the beginning of the string. + +``` +hello -> 5:hello, +foo! -> 4:foo!, +こんにちは -> 15:こんにちは, +``` + +They can be used to encode e.g. lists by simply concatenating and reading them in one-by-one. + +If you need a more complex encoding, you could start encoding e.g. tuples as netstrings-in-netstrings, or you could use [`netencode`](../netencode/README.md) instead, which is what-if-json-but-netstrings, and takes the idea of netstrings to their logical conclusion. + +Resources: + +Spec: http://cr.yp.to/proto/netstrings.txt +Wiki: https://en.wikipedia.org/wiki/Netstring diff --git a/users/Profpatsch/netstring/default.nix b/users/Profpatsch/netstring/default.nix new file mode 100644 index 000000000000..dcc29d7e6fec --- /dev/null +++ b/users/Profpatsch/netstring/default.nix @@ -0,0 +1,63 @@ +{ lib, pkgs, depot, ... }: +let + toNetstring = s: + "${toString (builtins.stringLength s)}:${s},"; + + toNetstringList = xs: + lib.concatStrings (map toNetstring xs); + + toNetstringKeyVal = attrs: + lib.concatStrings + (lib.mapAttrsToList + (k: v: toNetstring (toNetstring k + toNetstring v)) + attrs); + + python-netstring = depot.users.Profpatsch.writers.python3Lib { + name = "netstring"; + } '' + def read_netstring(bytes): + (int_length, rest) = bytes.split(sep=b':', maxsplit=1) + val = rest[:int(int_length)] + # has to end on a , + assert(rest[len(val)] == ord(',')) + return (val, rest[len(val) + 1:]) + + def read_netstring_key_val(bytes): + (keyvalnet, rest) = read_netstring(bytes) + (key, valnet) = read_netstring(keyvalnet) + (val, nothing) = read_netstring(valnet) + assert(nothing == b"") + return (key, val, rest) + + def read_netstring_key_val_list(bytes): + rest = bytes + res = {} + while rest != b"": + (key, val, r) = read_netstring_key_val(rest) + rest = r + res[key] = val + return res + ''; + + rust-netstring = depot.nix.writers.rustSimpleLib { + name = "netstring"; + } '' + pub fn to_netstring(s: &[u8]) -> Vec<u8> { + let len = s.len(); + // length of the integer as ascii + let i_len = ((len as f64).log10() as usize) + 1; + let ns_len = i_len + 1 + len + 1; + let mut res = Vec::with_capacity(ns_len); + res.extend_from_slice(format!("{}:", len).as_bytes()); + res.extend_from_slice(s); + res.push(b','); + res + } + ''; + +in depot.nix.readTree.drvTargets { + inherit + python-netstring + rust-netstring + ; +} diff --git a/users/Profpatsch/netstring/tests/default.nix b/users/Profpatsch/netstring/tests/default.nix new file mode 100644 index 000000000000..710ba3d30526 --- /dev/null +++ b/users/Profpatsch/netstring/tests/default.nix @@ -0,0 +1,61 @@ +{ depot, lib, pkgs, ... }: + +let + + python-netstring-test = depot.users.Profpatsch.writers.python3 { + name = "python-netstring-test"; + libraries = p: [ + depot.users.Profpatsch.netstring.python-netstring + ]; + } '' + import netstring + + def assEq(left, right): + assert left == right, "{} /= {}".format(str(left), str(right)) + + assEq( + netstring.read_netstring(b"""${depot.nix.netstring.fromString "hi!"}"""), + (b"hi!", b"") + ) + + assEq( + netstring.read_netstring_key_val( + b"""${depot.nix.netstring.attrsToKeyValList { foo = "42"; }}""" + ), + (b'foo', b'42', b"") + ) + + assEq( + netstring.read_netstring_key_val_list( + b"""${depot.nix.netstring.attrsToKeyValList { foo = "42"; bar = "hi"; }}""" + ), + { b'foo': b'42', b'bar': b'hi' } + ) + ''; + + rust-netstring-test = depot.nix.writers.rustSimple { + name = "rust-netstring-test"; + dependencies = [ + depot.users.Profpatsch.netstring.rust-netstring + ]; + } '' + extern crate netstring; + + fn main() { + assert_eq!( + std::str::from_utf8(&netstring::to_netstring(b"hello")).unwrap(), + r##"${depot.nix.netstring.fromString "hello"}"## + ); + assert_eq!( + std::str::from_utf8(&netstring::to_netstring("こんにちは".as_bytes())).unwrap(), + r##"${depot.nix.netstring.fromString "こんにちは"}"## + ); + } + ''; + +in depot.nix.readTree.drvTargets { + inherit + python-netstring-test + rust-netstring-test + ; +} diff --git a/users/Profpatsch/nixpkgs-rewriter/MetaStdenvLib.hs b/users/Profpatsch/nixpkgs-rewriter/MetaStdenvLib.hs new file mode 100644 index 000000000000..3ed96a7b6eac --- /dev/null +++ b/users/Profpatsch/nixpkgs-rewriter/MetaStdenvLib.hs @@ -0,0 +1,80 @@ +{-# LANGUAGE PartialTypeSignatures #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE NamedFieldPuns #-} +import Nix.Parser +import Nix.Expr.Types +import Nix.Expr.Types.Annotated +import System.Environment (getArgs) +import System.Exit (die) +import Data.Fix (Fix(..)) +import qualified Data.Text as Text +import qualified Data.ByteString.Lazy.Char8 as BL +import qualified Data.Aeson as A +import qualified Data.Aeson.Encoding as A +import Data.Function ((&)) +import qualified System.IO as IO +import qualified Text.Megaparsec.Pos as MP + +main = do + (nixFile:_) <- getArgs + (parseNixFileLoc nixFile :: IO _) >>= \case + Failure err -> do + ePutStrLn $ show err + die "oh no" + Success expr -> do + case snd $ match expr of + NoArguments -> do + ePutStrLn $ "NoArguments in " <> nixFile + printPairs mempty + YesLib vars -> do + ePutStrLn $ "lib in " <> show vars <> " in " <> nixFile + printPairs mempty + NoLib vars srcSpan -> do + ePutStrLn $ nixFile <> " needs lib added" + printPairs + $ "fileName" A..= nixFile + <> "fromLine" A..= (srcSpan & spanBegin & sourceLine) + <> "fromColumn" A..= (srcSpan & spanBegin & sourceColumn) + <> "toLine" A..= (srcSpan & spanEnd & sourceLine) + <> "toColumn" A..= (srcSpan & spanEnd & sourceColumn) + +printPairs pairs = BL.putStrLn $ A.encodingToLazyByteString $ A.pairs pairs + +ePutStrLn = IO.hPutStrLn IO.stderr + +data Descend = YesDesc | NoDesc + deriving Show +data Matched = NoArguments | NoLib [VarName] SrcSpan | YesLib [VarName] + deriving Show + +match :: Fix (Compose (Ann SrcSpan) NExprF) -> (Descend, Matched) +match = \case + (AnnE outerSpan (NAbs (ParamSet params _ _) (AnnE innerSpan _))) -> (NoDesc, + let vars = map fst params in + case (any (== "lib") vars) of + True -> YesLib vars + False -> + -- The span of the arglist is from the beginning of the match + -- to the beginning of the inner expression + let varSpan = SrcSpan + { spanBegin = outerSpan & spanBegin + -- -1 to prevent the spans from overlapping + , spanEnd = sourcePosMinus1 (innerSpan & spanBegin) } + in NoLib vars varSpan) + _ -> (NoDesc, NoArguments) + +-- | Remove one from a source positon. +-- +-- That means if the current position is at the very beginning of a line, +-- jump to the previous line. +sourcePosMinus1 :: SourcePos -> SourcePos +sourcePosMinus1 src@(SourcePos { sourceLine, sourceColumn }) = + let + col = MP.mkPos $ max (MP.unPos sourceColumn - 1) 1 + line = MP.mkPos $ case MP.unPos sourceColumn of + 1 -> max (MP.unPos sourceLine - 1) 1 + _ -> MP.unPos sourceLine + in src + { sourceLine = line + , sourceColumn = col } diff --git a/users/Profpatsch/nixpkgs-rewriter/default.nix b/users/Profpatsch/nixpkgs-rewriter/default.nix new file mode 100644 index 000000000000..787162d4973a --- /dev/null +++ b/users/Profpatsch/nixpkgs-rewriter/default.nix @@ -0,0 +1,112 @@ +{ depot, pkgs, ... }: +let + inherit (depot.nix) + writeExecline + ; + inherit (depot.users.Profpatsch.lib) + debugExec + ; + + bins = depot.nix.getBins pkgs.coreutils [ "head" "shuf" ] + // depot.nix.getBins pkgs.jq [ "jq" ] + // depot.nix.getBins pkgs.findutils [ "xargs" ] + // depot.nix.getBins pkgs.gnused [ "sed" ] + ; + + export-json-object = pkgs.writers.writePython3 "export-json-object" {} '' + import json + import sys + import os + + d = json.load(sys.stdin) + + if d == {}: + sys.exit(0) + + for k, v in d.items(): + os.environ[k] = str(v) + + os.execvp(sys.argv[1], sys.argv[1:]) + ''; + + meta-stdenv-lib = pkgs.writers.writeHaskell "meta-stdenv-lib" { + libraries = [ + pkgs.haskellPackages.hnix + pkgs.haskellPackages.aeson + ]; + } ./MetaStdenvLib.hs; + + replace-between-lines = writeExecline "replace-between-lines" { readNArgs = 1; } [ + "importas" "-ui" "file" "fileName" + "importas" "-ui" "from" "fromLine" + "importas" "-ui" "to" "toLine" + "if" [ depot.tools.eprintf "%s-%s\n" "$from" "$to" ] + (debugExec "adding lib") + bins.sed + "-e" "\${from},\${to} \${1}" + "-i" "$file" + ]; + + add-lib-if-necessary = writeExecline "add-lib-if-necessary" { readNArgs = 1; } [ + "pipeline" [ meta-stdenv-lib "$1" ] + export-json-object + # first replace any stdenv.lib mentions in the arg header + # if this is not done, the replace below kills these. + # Since we want it anyway ultimately, let’s do it here. + "if" [ replace-between-lines "s/stdenv\.lib/lib/" ] + # then add the lib argument + # (has to be before stdenv, otherwise default arguments might be in the way) + replace-between-lines "s/stdenv/lib, stdenv/" + ]; + + metaString = ''meta = with stdenv.lib; {''; + + replace-stdenv-lib = pkgs.writers.writeBash "replace-stdenv-lib" '' + set -euo pipefail + sourceDir="$1" + for file in $( + ${pkgs.ripgrep}/bin/rg \ + --files-with-matches \ + --fixed-strings \ + -e '${metaString}' \ + "$sourceDir" + ) + do + echo "replacing stdenv.lib meta in $file" >&2 + ${bins.sed} -e '/${metaString}/ s/stdenv.lib/lib/' \ + -i "$file" + ${add-lib-if-necessary} "$file" + done + ''; + + instantiate-nixpkgs-randomly = writeExecline "instantiate-nixpkgs-randomly" { readNArgs = 1; } [ + "export" "NIXPKGS_ALLOW_BROKEN" "1" + "export" "NIXPKGS_ALLOW_UNFREE" "1" + "export" "NIXPKGS_ALLOW_INSECURE" "1" + "export" "NIXPKGS_ALLOW_UNSUPPORTED_SYSTEM" "1" + "pipeline" [ + "nix" + "eval" + "--raw" + ''( + let pkgs = import ''${1} {}; + in builtins.toJSON (builtins.attrNames pkgs) + )'' + ] + "pipeline" [ bins.jq "-r" ".[]" ] + "pipeline" [ bins.shuf ] + "pipeline" [ bins.head "-n" "1000" ] + bins.xargs "-I" "{}" "-n1" + "if" [ depot.tools.eprintf "instantiating %s\n" "{}" ] + "nix-instantiate" "$1" "-A" "{}" + ]; + +in depot.nix.readTree.drvTargets { + inherit + instantiate-nixpkgs-randomly + # requires hnix, which we don’t want in tvl for now + # uncomment manually if you want to use it. + # meta-stdenv-lib + # replace-stdenv-lib + ; +} diff --git a/users/Profpatsch/read-http.nix b/users/Profpatsch/read-http.nix new file mode 100644 index 000000000000..854a11b7d099 --- /dev/null +++ b/users/Profpatsch/read-http.nix @@ -0,0 +1,16 @@ +{ depot, pkgs, ... }: + +let + + read-http = depot.nix.writers.rustSimple { + name = "read-http"; + dependencies = [ + depot.third_party.rust-crates.ascii + depot.third_party.rust-crates.httparse + depot.users.Profpatsch.netencode.netencode-rs + depot.users.Profpatsch.arglib.netencode.rust + depot.users.Profpatsch.execline.exec-helpers + ]; + } (builtins.readFile ./read-http.rs); + +in read-http diff --git a/users/Profpatsch/read-http.rs b/users/Profpatsch/read-http.rs new file mode 100644 index 000000000000..50ff663b994a --- /dev/null +++ b/users/Profpatsch/read-http.rs @@ -0,0 +1,211 @@ +extern crate httparse; +extern crate netencode; +extern crate arglib_netencode; +extern crate ascii; +extern crate exec_helpers; + +use std::os::unix::io::FromRawFd; +use std::io::Read; +use std::io::Write; +use std::collections::HashMap; +use exec_helpers::{die_user_error, die_expected_error, die_temporary}; + +use netencode::{U, T, dec}; +use netencode::dec::Decoder; + +enum What { + Request, + Response +} + +// reads a http request (stdin), and writes all headers to stdout, as netencoded record. +// The keys are text, but can be lists of text iff headers appear multiple times, so beware. +fn main() -> std::io::Result<()> { + + exec_helpers::no_args("read-http"); + + let args = dec::RecordDot { + field: "what", + inner: dec::OneOf { + list: vec!["request", "response"], + inner: dec::Text + } + }; + let what : What = match args.dec(arglib_netencode::arglib_netencode("read-http", None).to_u()) { + Ok("request") => What::Request, + Ok("response") => What::Response, + Ok(v) => panic!("shouldn’t happen!, value was: {}", v), + Err(dec::DecodeError(err)) => die_user_error("read-http", err), + }; + + fn read_stdin_to_complete<F>(mut parse: F) -> () + where F: FnMut(&[u8]) -> httparse::Result<usize> + { + let mut res = httparse::Status::Partial; + loop { + if let httparse::Status::Complete(_) = res { + return; + } + let mut buf = [0; 2048]; + match std::io::stdin().read(&mut buf[..]) { + Ok(size) => if size == 0 { + break; + }, + Err(err) => die_temporary("read-http", format!("could not read from stdin, {:?}", err)) + } + match parse(&buf) { + Ok(status) => { + res = status; + } + Err(err) => die_temporary("read-http", format!("httparse parsing failed: {:#?}", err)) + } + } + } + + fn normalize_headers<'a>(headers: &'a [httparse::Header]) -> HashMap<String, U<'a>> { + let mut res = HashMap::new(); + for httparse::Header { name, value } in headers { + let val = ascii::AsciiStr::from_ascii(*value) + .expect(&format!("read-http: we require header values to be ASCII, but the header {} was {:?}", name, value)) + .as_str(); + // lowercase the header names, since the standard doesn’t care + // and we want unique strings to match against + let name_lower = name.to_lowercase(); + match res.insert(name_lower, U::Text(val)) { + None => (), + Some(U::Text(t)) => { + let name_lower = name.to_lowercase(); + let _ = res.insert(name_lower, U::List(vec![U::Text(t), U::Text(val)])); + () + }, + Some(U::List(mut l)) => { + let name_lower = name.to_lowercase(); + l.push(U::Text(val)); + let _ = res.insert(name_lower, U::List(l)); + () + }, + Some(o) => panic!("read-http: header not text nor list: {:?}", o), + } + } + res + } + + // tries to read until the end of the http header (deliniated by two newlines "\r\n\r\n") + fn read_till_end_of_header<R: Read>(buf: &mut Vec<u8>, reader: R) -> Option<()> { + let mut chonker = Chunkyboi::new(reader, 4096); + loop { + // TODO: attacker can send looooong input, set upper maximum + match chonker.next() { + Some(Ok(chunk)) => { + buf.extend_from_slice(&chunk); + if chunk.windows(4).any(|c| c == b"\r\n\r\n" ) { + return Some(()); + } + }, + Some(Err(err)) => die_temporary("read-http", format!("error reading from stdin: {:?}", err)), + None => return None + } + } + } + + // max header size chosen arbitrarily + let mut headers = [httparse::EMPTY_HEADER; 128]; + let stdin = std::io::stdin(); + + match what { + Request => { + let mut req = httparse::Request::new(&mut headers); + let mut buf: Vec<u8> = vec![]; + match read_till_end_of_header(&mut buf, stdin.lock()) { + Some(()) => match req.parse(&buf) { + Ok(httparse::Status::Complete(_body_start)) => {}, + Ok(httparse::Status::Partial) => die_expected_error("read-http", "httparse should have gotten a full header"), + Err(err) => die_expected_error("read-http", format!("httparse response parsing failed: {:#?}", err)) + }, + None => die_expected_error("read-http", format!("httparse end of stdin reached before able to parse request headers")) + } + let method = req.method.expect("method must be filled on complete parse"); + let path = req.path.expect("path must be filled on complete parse"); + write_dict_req(method, path, &normalize_headers(req.headers)) + }, + Response => { + let mut resp = httparse::Response::new(&mut headers); + let mut buf: Vec<u8> = vec![]; + match read_till_end_of_header(&mut buf, stdin.lock()) { + Some(()) => match resp.parse(&buf) { + Ok(httparse::Status::Complete(_body_start)) => {}, + Ok(httparse::Status::Partial) => die_expected_error("read-http", "httparse should have gotten a full header"), + Err(err) => die_expected_error("read-http", format!("httparse response parsing failed: {:#?}", err)) + }, + None => die_expected_error("read-http", format!("httparse end of stdin reached before able to parse response headers")) + } + let code = resp.code.expect("code must be filled on complete parse"); + let reason = resp.reason.expect("reason must be filled on complete parse"); + write_dict_resp(code, reason, &normalize_headers(resp.headers)) + } + } +} + +fn write_dict_req<'a, 'buf>(method: &'buf str, path: &'buf str, headers: &'a HashMap<String, U<'a>>) -> std::io::Result<()> { + let mut http = vec![ + ("method", U::Text(method)), + ("path", U::Text(path)), + ].into_iter().collect(); + write_dict(http, headers) +} + +fn write_dict_resp<'a, 'buf>(code: u16, reason: &'buf str, headers: &'a HashMap<String, U<'a>>) -> std::io::Result<()> { + let mut http = vec![ + ("status", U::N6(code as u64)), + ("status-text", U::Text(reason)), + ].into_iter().collect(); + write_dict(http, headers) +} + + +fn write_dict<'buf, 'a>(mut http: HashMap<&str, U<'a>>, headers: &'a HashMap<String, U<'a>>) -> std::io::Result<()> { + match http.insert("headers", U::Record( + headers.iter().map(|(k,v)| (k.as_str(), v.clone())).collect() + )) { + None => (), + Some(_) => panic!("read-http: headers already in dict"), + }; + netencode::encode( + &mut std::io::stdout(), + &U::Record(http) + )?; + Ok(()) +} + + +// iter helper + +struct Chunkyboi<T> { + inner: T, + buf: Vec<u8>, +} + +impl<R: Read> Chunkyboi<R> { + fn new(inner: R, chunksize: usize) -> Self { + let buf = vec![0; chunksize]; + Chunkyboi { + inner, + buf + } + } +} + +impl<R: Read> Iterator for Chunkyboi<R> { + type Item = std::io::Result<Vec<u8>>; + + fn next(&mut self) -> Option<std::io::Result<Vec<u8>>> { + match self.inner.read(&mut self.buf) { + Ok(0) => None, + Ok(read) => { + // clone a new buffer so we can reuse the internal one + Some(Ok(self.buf[..read].to_owned())) + } + Err(err) => Some(Err(err)) + } + } +} diff --git a/users/Profpatsch/reverse-haskell-deps.hs b/users/Profpatsch/reverse-haskell-deps.hs new file mode 100644 index 000000000000..6b644df9ecc6 --- /dev/null +++ b/users/Profpatsch/reverse-haskell-deps.hs @@ -0,0 +1,72 @@ +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE MultiWayIf #-} +{-# LANGUAGE ScopedTypeVariables #-} +import qualified Text.HTML.TagSoup as Tag +import qualified Data.Text as Text +import Data.Text (Text) +import qualified Data.List as List +import Data.Maybe +import Text.Nicify +import qualified Text.Read as Read +import Numeric.Natural +import Data.Either +import qualified Data.ByteString as ByteString +import qualified Data.Text.Encoding + +parseNat :: Text.Text -> Maybe Natural +parseNat = Read.readMaybe . Text.unpack + +printNice :: Show a => a -> IO () +printNice = putStrLn . nicify . show + +type Tag = Tag.Tag Text.Text + +main = do + reverseHtml <- readStdinUtf8 + printNice $ List.sortOn snd $ packagesAndReverseDeps reverseHtml + + where + readStdinUtf8 = Data.Text.Encoding.decodeUtf8 <$> ByteString.getContents + +-- | reads the table provided by https://packdeps.haskellers.com/reverse +-- figuring out all sections (starting with the link to the package name), +-- then figuring out the name of the package and the first column, +-- which is the number of reverse dependencies of the package +packagesAndReverseDeps reverseHtml = do + let tags = Tag.parseTags reverseHtml + let sections = Tag.partitions (isJust . reverseLink) tags + let sectionNames = map (fromJust . reverseLink . head) sections + mapMaybe + (\(name :: Text.Text, sect) -> do + reverseDeps <- firstNaturalNumber sect + pure (sectionPackageName name sect, reverseDeps) :: Maybe (Text.Text, Natural)) + $ zip sectionNames sections + + + where + reverseLink = \case + Tag.TagOpen "a" attrs -> mapFind attrReverseLink attrs + _ -> Nothing + + attrReverseLink = \case + ("href", lnk) -> if + | "packdeps.haskellers.com/reverse/" `Text.isInfixOf` lnk -> Just lnk + | otherwise -> Nothing + _ -> Nothing + + sectionPackageName :: Text -> [Tag] -> Text + sectionPackageName sectionName = \case + (_: Tag.TagText name : _) -> name + (_: el : _) -> sectionName + xs -> sectionName + + + firstNaturalNumber :: [Tag] -> Maybe Natural + firstNaturalNumber = + mapFind (\case + Tag.TagText t -> parseNat t + _ -> Nothing) + + mapFind :: (a -> Maybe b) -> [a] -> Maybe b + mapFind f xs = fromJust . f <$> List.find (isJust . f) xs diff --git a/users/Profpatsch/reverse-haskell-deps.nix b/users/Profpatsch/reverse-haskell-deps.nix new file mode 100644 index 000000000000..b47347ea9fea --- /dev/null +++ b/users/Profpatsch/reverse-haskell-deps.nix @@ -0,0 +1,26 @@ +{ depot, pkgs, ... }: + +# Parses https://packdeps.haskellers.com/reverse +# and outputs the amount of reverse dependencies of each hackage package. + +let + + rev = depot.nix.writeExecline "reverse-haskell-deps" {} [ + "pipeline" [ + "${pkgs.curl}/bin/curl" "-L" "https://packdeps.haskellers.com/reverse" + ] + rev-hs + + ]; + + rev-hs = pkgs.writers.writeHaskell "revers-haskell-deps-hs" { + libraries = [ + pkgs.haskellPackages.nicify-lib + pkgs.haskellPackages.tagsoup + ]; + + } + ./reverse-haskell-deps.hs; + + +in rev diff --git a/users/Profpatsch/struct-edit/default.nix b/users/Profpatsch/struct-edit/default.nix new file mode 100644 index 000000000000..970cdd4d028b --- /dev/null +++ b/users/Profpatsch/struct-edit/default.nix @@ -0,0 +1,13 @@ +{ depot, ... }: +depot.nix.buildGo.program { + name = "struct-edit"; + srcs = [ + ./main.go + ]; + deps = [ + depot.third_party.gopkgs."github.com".charmbracelet.bubbletea + depot.third_party.gopkgs."github.com".charmbracelet.lipgloss + depot.third_party.gopkgs."github.com".muesli.termenv + depot.third_party.gopkgs."github.com".mattn.go-isatty + ]; +} diff --git a/users/Profpatsch/struct-edit/main.go b/users/Profpatsch/struct-edit/main.go new file mode 100644 index 000000000000..7e43074266d9 --- /dev/null +++ b/users/Profpatsch/struct-edit/main.go @@ -0,0 +1,431 @@ +package main + +import ( + json "encoding/json" + "fmt" + "log" + "os" + "strings" + "sort" + + tea "github.com/charmbracelet/bubbletea" + lipgloss "github.com/charmbracelet/lipgloss" + // termenv "github.com/muesli/termenv" + // isatty "github.com/mattn/go-isatty" +) + +// Keeps the full data structure and a path that indexes our current position into it. +type model struct { + path []index + data val +} + +// an index into a value, uint for lists and string for maps. +// nil for any scalar value. +// TODO: use an actual interface for these +type index interface{} + +/// recursive value that we can represent. +type val struct { + // the “type” of value; see tag const belove + tag tag + // last known position of our cursor + last_index index + // documentation (TODO) + doc string + // the actual value; + // the actual structure is behind a pointer so we can replace the struct. + // determined by the tag + // tagString -> *string + // tagFloat -> *float64 + // tagList -> *[]val + // tagMap -> *map[string]val + val interface{} +} + +type tag string + +const ( + tagString tag = "string" + tagFloat tag = "float" + tagList tag = "list" + tagMap tag = "map" +) + +// print a value, flat +func (v val) Render() string { + s := "" + switch v.tag { + case tagString: + s += *v.val.(*string) + case tagFloat: + s += fmt.Sprint(*v.val.(*float64)) + case tagList: + s += "[ " + vs := []string{} + for _, enum := range v.enumerate() { + vs = append(vs, enum.v.Render()) + } + s += strings.Join(vs, ", ") + s += " ]" + case tagMap: + s += "{ " + vs := []string{} + for _, enum := range v.enumerate() { + vs = append(vs, fmt.Sprintf("%s: %s", enum.i.(string), enum.v.Render())) + } + s += strings.Join(vs, ", ") + s += " }" + default: + s += fmt.Sprintf("<unknown: %v>", v) + } + return s +} + +// render an index, depending on the type +func renderIndex(i index) (s string) { + switch i := i.(type) { + case nil: + s = "" + // list index + case uint: + s = "*" + // map index + case string: + s = i + ":" + } + return +} + +// take an arbitrary (within restrictions) go value and construct a val from it +func makeVal(i interface{}) val { + var v val + switch i := i.(type) { + case string: + v = val{ + tag: tagString, + last_index: index(nil), + doc: "", + val: &i, + } + case float64: + v = val{ + tag: tagFloat, + last_index: index(nil), + doc: "", + val: &i, + } + case []interface{}: + ls := []val{} + for _, i := range i { + ls = append(ls, makeVal(i)) + } + v = val{ + tag: tagList, + last_index: pos1Inner(tagList, &ls), + doc: "", + val: &ls, + } + case map[string]interface{}: + ls := map[string]val{} + for k, i := range i { + ls[k] = makeVal(i) + } + v = val{ + tag: tagMap, + last_index: pos1Inner(tagMap, &ls), + doc: "", + val: &ls, + } + default: + log.Fatalf("makeVal: cannot read json of type %T", i) + } + return v +} + +// return an index that points at the first entry in val +func (v val) pos1() index { + return v.enumerate()[0].i +} + +func pos1Inner(tag tag, v interface{}) index { + return enumerateInner(tag, v)[0].i +} + +type enumerate struct { + i index + v val +} + +// enumerate gives us a stable ordering of elements in this val. +// for scalars it’s just a nil index & the val itself. +// Guaranteed to always return at least one element. +func (v val) enumerate() (e []enumerate) { + e = enumerateInner(v.tag, v.val) + if e == nil { + e = append(e, enumerate{ + i: nil, + v: v, + }) + } + return +} + +// like enumerate, but returns an empty slice for scalars without inner vals. +func enumerateInner(tag tag, v interface{}) (e []enumerate) { + switch tag { + case tagString: + fallthrough + case tagFloat: + e = nil + case tagList: + for i, v := range *v.(*[]val) { + e = append(e, enumerate{i: index(uint(i)), v: v}) + } + case tagMap: + // map sorting order is not stable (actually randomized thank jabber) + // so let’s sort them + keys := []string{} + m := *v.(*map[string]val) + for k, _ := range m { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + e = append(e, enumerate{i: index(k), v: m[k]}) + } + default: + log.Fatalf("unknown val tag %s, %v", tag, v) + } + return +} + +func (m model) PathString() string { + s := "/ " + var is []string + for _, v := range m.path { + is = append(is, fmt.Sprintf("%v", v)) + } + s += strings.Join(is, " / ") + return s +} + +// walk the given path down in data, to get the value at that point. +// Assumes that all path indexes are valid indexes into data. +// Returns a pointer to the value at point, in order to be able to change it. +func walk(data *val, path []index) (*val, bool, error) { + res := data + atPath := func(index int) string { + return fmt.Sprintf("at path %v", path[:index+1]) + } + errf := func(ty string, val interface{}, index int) error { + return fmt.Errorf("walk: can’t walk into %s %v %s", ty, val, atPath(index)) + } + for i, p := range path { + switch res.tag { + case tagString: + return nil, true, nil + case tagFloat: + return nil, true, nil + case tagList: + switch p := p.(type) { + case uint: + list := *res.val.(*[]val) + if int(p) >= len(list) || p < 0 { + return nil, false, fmt.Errorf("index out of bounds %s", atPath(i)) + } + res = &list[p] + default: + return nil, false, fmt.Errorf("not a list index %s", atPath(i)) + } + case tagMap: + switch p := p.(type) { + case string: + m := *res.val.(*map[string]val) + if a, ok := m[p]; ok { + res = &a + } else { + return nil, false, fmt.Errorf("index %s not in map %s", p, atPath(i)) + } + default: + return nil, false, fmt.Errorf("not a map index %v %s", p, atPath(i)) + } + + default: + return nil, false, errf(string(res.tag), res.val, i) + } + } + return res, false, nil +} + +// descend into the selected index. Assumes that the index is valid. +// Will not descend into scalars. +func (m model) descend() (model, error) { + // TODO: two walks?! + this, _, err := walk(&m.data, m.path) + if err != nil { + return m, err + } + newPath := append(m.path, this.last_index) + _, bounce, err := walk(&m.data, newPath) + if err != nil { + return m, err + } + // only descend if we *can* + if !bounce { + m.path = newPath + } + return m, nil +} + +// ascend to one level up. stops at the root. +func (m model) ascend() (model, error) { + if len(m.path) > 0 { + m.path = m.path[:len(m.path)-1] + _, _, err := walk(&m.data, m.path) + return m, err + } + return m, nil +} + +/// go to the next item, or wraparound +func (min model) next() (m model, err error) { + m = min + this, _, err := walk(&m.data, m.path) + if err != nil { + return + } + enumL := this.enumerate() + setNext := false + for _, enum := range enumL { + if setNext { + this.last_index = enum.i + setNext = false + break + } + if enum.i == this.last_index { + setNext = true + } + } + // wraparound + if setNext { + this.last_index = enumL[0].i + } + return +} + +/// go to the previous item, or wraparound +func (min model) prev() (m model, err error) { + m = min + this, _, err := walk(&m.data, m.path) + if err != nil { + return + } + enumL := this.enumerate() + // last element, wraparound + prevIndex := enumL[len(enumL)-1].i + for _, enum := range enumL { + if enum.i == this.last_index { + this.last_index = prevIndex + break + } + prevIndex = enum.i + } + return +} + +/// bubbletea implementations + +func (m model) Init() tea.Cmd { + return nil +} + +func initialModel(v interface{}) model { + val := makeVal(v) + return model{ + path: []index{}, + data: val, + } +} + +func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + var err error + switch msg := msg.(type) { + case tea.KeyMsg: + switch msg.String() { + case "ctrl+c", "q": + return m, tea.Quit + + case "up": + m, err = m.prev() + + case "down": + m, err = m.next() + + case "right": + m, err = m.descend() + + case "left": + m, err = m.ascend() + + // case "enter": + // _, ok := m.selected[m.cursor] + // if ok { + // delete(m.selected, m.cursor) + // } else { + // m.selected[m.cursor] = struct{}{} + // } + } + + } + if err != nil { + log.Fatal(err) + } + return m, nil +} + +var pathColor = lipgloss.NewStyle(). + // light blue + Foreground(lipgloss.Color("12")) + +var selectedColor = lipgloss.NewStyle(). + Bold(true) + +func (m model) View() string { + s := pathColor.Render(m.PathString()) + cur, _, err := walk(&m.data, m.path) + if err != nil { + log.Fatal(err) + } + s += cur.doc + "\n" + s += "\n" + for _, enum := range cur.enumerate() { + is := renderIndex(enum.i) + if is != "" { + s += is + " " + } + if enum.i == cur.last_index { + s += selectedColor.Render(enum.v.Render()) + } else { + s += enum.v.Render() + } + s += "\n" + } + + // s += fmt.Sprintf("%v\n", m) + // s += fmt.Sprintf("%v\n", cur) + + return s +} + +func main() { + var input interface{} + err := json.NewDecoder(os.Stdin).Decode(&input) + if err != nil { + log.Fatal("json from stdin: ", err) + } + p := tea.NewProgram(initialModel(input)) + if err := p.Start(); err != nil { + log.Fatal("bubbletea TUI error: ", err) + } +} diff --git a/users/Profpatsch/tree-sitter.nix b/users/Profpatsch/tree-sitter.nix new file mode 100644 index 000000000000..4f81b8e7a77c --- /dev/null +++ b/users/Profpatsch/tree-sitter.nix @@ -0,0 +1,179 @@ +{ depot, pkgs, lib, ... }: + +let + bins = depot.nix.getBins pkgs.coreutils [ "head" "printf" "cat" ] + // depot.nix.getBins pkgs.ncurses [ "tput" ] + // depot.nix.getBins pkgs.bc [ "bc" ] + // depot.nix.getBins pkgs.ocamlPackages.sexp [ "sexp" ]; + + print-ast = depot.nix.writers.rustSimple { + name = "print-ast"; + dependencies = with depot.third_party.rust-crates; [ + libloading + tree-sitter + ]; + } '' + extern crate libloading; + extern crate tree_sitter; + use std::mem; + use std::io::{Read}; + use libloading::{Library, Symbol}; + use tree_sitter::{Language, Parser}; + + /// Load the shared lib FILE and return the language under SYMBOL-NAME. + /// Inspired by the rust source of emacs-tree-sitter. + fn _load_language(file: String, symbol_name: String) -> Result<Language, libloading::Error> { + let lib = Library::new(file)?; + let tree_sitter_lang: Symbol<'_, unsafe extern "C" fn() -> _> = + unsafe { lib.get(symbol_name.as_bytes())? }; + let language: Language = unsafe { tree_sitter_lang() }; + // Avoid segmentation fault by not unloading the lib, as language is a static piece of data. + // TODO: Attach an Rc<Library> to Language instead. + mem::forget(lib); + Ok(language) + } + + fn main() { + let mut args = std::env::args(); + let so = args.nth(1).unwrap(); + let symbol_name = args.nth(0).unwrap(); + let file = args.nth(0).unwrap(); + let mut parser = Parser::new(); + let lang = _load_language(so, symbol_name).unwrap(); + parser.set_language(lang).unwrap(); + let bytes = std::fs::read(&file).unwrap(); + print!("{}", parser.parse(&bytes, None).unwrap().root_node().to_sexp()); + } + + + ''; + + tree-sitter-nix = buildTreeSitterGrammar { + language = "tree-sitter-nix"; + source = pkgs.fetchFromGitHub { + owner = "cstrahan"; + repo = "tree-sitter-nix"; + rev = "791b5ff0e4f0da358cbb941788b78d436a2ca621"; + sha256 = "1y5b3wh3fcmbgq8r2i97likzfp1zp02m58zacw5a1cjqs5raqz66"; + }; + }; + + watch-file-modified = depot.nix.writers.rustSimple { + name = "watch-file-modified"; + dependencies = [ + depot.third_party.rust-crates.inotify + depot.users.Profpatsch.netstring.rust-netstring + ]; + } '' + extern crate inotify; + extern crate netstring; + use inotify::{EventMask, WatchMask, Inotify}; + use std::io::Write; + + fn main() { + let mut inotify = Inotify::init() + .expect("Failed to initialize inotify"); + + let file = std::env::args().nth(1).unwrap(); + + let file_watch = inotify + .add_watch( + &file, + WatchMask::MODIFY + ) + .expect("Failed to add inotify watch"); + + let mut buffer = [0u8; 4096]; + loop { + let events = inotify + .read_events_blocking(&mut buffer) + .expect("Failed to read inotify events"); + + for event in events { + if event.wd == file_watch { + std::io::stdout().write(&netstring::to_netstring(file.as_bytes())); + std::io::stdout().flush(); + } + } + } + } + + ''; + + # clear screen and set LINES and COLUMNS to terminal height & width + clear-screen = depot.nix.writeExecline "clear-screen" {} [ + "if" [ bins.tput "clear" ] + "backtick" "-in" "LINES" [ bins.tput "lines" ] + "backtick" "-in" "COLUMNS" [ bins.tput "cols" ] + "$@" + ]; + + print-nix-file = depot.nix.writeExecline "print-nix-file" { readNArgs = 1; } [ + "pipeline" [ print-ast "${tree-sitter-nix}/parser" "tree_sitter_nix" "$1" ] + "pipeline" [ bins.sexp "print" ] + clear-screen + "importas" "-ui" "lines" "LINES" + "backtick" "-in" "ls" [ + "pipeline" + # when you pull out bc to decrement an integer it’s time to switch to python lol + [ bins.printf "x=%s; --x\n" "$lines" ] + bins.bc + ] + "importas" "-ui" "l" "ls" + bins.head "-n\${l}" + ]; + + print-nix-file-on-update = depot.nix.writeExecline "print-nix-file-on-update" { readNArgs = 1; } [ + "if" [ print-nix-file "$1" ] + "pipeline" [ watch-file-modified "$1" ] + "forstdin" "-d" "" "file" + "importas" "file" "file" + print-nix-file "$file" + ]; + + # copied from nixpkgs + buildTreeSitterGrammar = + { + # language name + language + # source for the language grammar + , source + }: + + pkgs.stdenv.mkDerivation { + + pname = "${language}-grammar"; + inherit (pkgs.tree-sitter) version; + + src = source; + + buildInputs = [ pkgs.tree-sitter ]; + + dontUnpack = true; + configurePhase= ":"; + buildPhase = '' + runHook preBuild + scanner_cc="$src/src/scanner.cc" + if [ ! -f "$scanner_cc" ]; then + scanner_cc="" + fi + $CXX -I$src/src/ -c $scanner_cc + $CC -I$src/src/ -shared -o parser -Os scanner.o $src/src/parser.c -lstdc++ + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir $out + mv parser $out/ + runHook postInstall + ''; + }; + +in depot.nix.readTree.drvTargets { + inherit + print-ast + tree-sitter-nix + print-nix-file-on-update + watch-file-modified + ; +} diff --git a/users/Profpatsch/writers/default.nix b/users/Profpatsch/writers/default.nix new file mode 100644 index 000000000000..3151a9d3bd44 --- /dev/null +++ b/users/Profpatsch/writers/default.nix @@ -0,0 +1,76 @@ +{ depot, pkgs, lib, ... }: +let + bins = depot.nix.getBins pkgs.s6-portable-utils ["s6-mkdir" "s6-cat" "s6-ln" "s6-ls" "s6-touch" ] + // depot.nix.getBins pkgs.coreutils ["printf" ]; + + inherit (depot.nix.yants) defun struct restrict attrs list string drv any; + + inherit (depot.nix) drvSeqL; + + FlakeError = + restrict + "flake error" + (s: lib.any (prefix: (builtins.substring 0 1 s) == prefix) + [ "E" "W" ]) + string; + Libraries = defun [ (attrs any) (list drv) ]; + + python3 = { + name, + libraries ? (_: []), + flakeIgnore ? [] + }: pkgs.writers.writePython3 name { + libraries = Libraries libraries pkgs.python3Packages; + flakeIgnore = + let ignoreTheseErrors = [ + # whitespace after { + "E201" + # whitespace before } + "E202" + # fuck 4-space indentation + "E121" "E111" + # who cares about blank lines … + # … at end of files + "W391" + # … between functions + "E302" "E305" + ]; + in list FlakeError (ignoreTheseErrors ++ flakeIgnore); + }; + + # TODO: add the same flake check as the pyhon3 writer + python3Lib = { name, libraries ? (_: []) }: moduleString: + let srcTree = depot.nix.runExecline.local name { stdin = moduleString; } [ + "importas" "out" "out" + "if" [ bins.s6-mkdir "-p" "\${out}/${name}" ] + "if" [ + "redirfd" "-w" "1" "\${out}/setup.py" + bins.printf '' + from distutils.core import setup + + setup( + name='%s', + packages=['%s'] + ) + '' name name + ] + "if" [ + # redirect stdin to the init py + "redirfd" "-w" "1" "\${out}/${name}/__init__.py" + bins.s6-cat + ] + ]; + in pkgs.python3Packages.buildPythonPackage { + inherit name; + src = srcTree; + propagatedBuildInputs = libraries pkgs.python3Packages; + doCheck = false; + }; + + +in { + inherit + python3 + python3Lib + ; +} diff --git a/users/Profpatsch/writers/tests/default.nix b/users/Profpatsch/writers/tests/default.nix new file mode 100644 index 000000000000..dc760af9e16e --- /dev/null +++ b/users/Profpatsch/writers/tests/default.nix @@ -0,0 +1,48 @@ +{ depot, pkgs, ... }: + +let + inherit (depot.users.Profpatsch.writers) + python3Lib + python3 + ; + + inherit (pkgs) + coreutils + ; + + run = drv: depot.nix.runExecline.local "run-${drv.name}" {} [ + "if" [ drv ] + "importas" "out" "out" + "${coreutils}/bin/touch" "$out" + ]; + + pythonTransitiveLib = python3Lib { + name = "transitive"; + } '' + def transitive(s): + return s + " 1 2 3" + ''; + + pythonTestLib = python3Lib { + name = "test_lib"; + libraries = _: [ pythonTransitiveLib ]; + } '' + import transitive + def test(): + return transitive.transitive("test") + ''; + + pythonWithLib = run (python3 { + name = "python-with-lib"; + libraries = _: [ pythonTestLib ]; + } '' + import test_lib + + assert(test_lib.test() == "test 1 2 3") + ''); + +in depot.nix.readTree.drvTargets { + inherit + pythonWithLib + ; +} |