diff options
Diffstat (limited to 'users/Profpatsch/blog')
| -rw-r--r-- | users/Profpatsch/blog/default.nix | 463 | ||||
| -rw-r--r-- | users/Profpatsch/blog/notes/preventing-oom.md | 33 | ||||
| -rw-r--r-- | users/Profpatsch/blog/notes/rust-string-conversions.md | 53 | ||||
| -rw-r--r-- | users/Profpatsch/blog/posts/2017-05-04-ligature-emulation-in-emacs.md | 123 |
4 files changed, 672 insertions, 0 deletions
diff --git a/users/Profpatsch/blog/default.nix b/users/Profpatsch/blog/default.nix new file mode 100644 index 000000000000..44a7a21aa3c3 --- /dev/null +++ b/users/Profpatsch/blog/default.nix @@ -0,0 +1,463 @@ +{ depot, pkgs, lib, ... }: + +let + bins = depot.nix.getBins pkgs.lowdown [ "lowdown" ] + // depot.nix.getBins pkgs.cdb [ "cdbget" "cdbmake" "cdbdump" ] + // depot.nix.getBins pkgs.coreutils [ "mv" "cat" "printf" "test" ] + // depot.nix.getBins pkgs.s6-networking [ "s6-tcpserver" ] + // depot.nix.getBins pkgs.time [ "time" ] + ; + + # / + # TODO: use + toplevel = [ + { + route = [ "notes" ]; + name = "Notes"; + page = { cssFile }: router cssFile; + } + { + route = [ "projects" ]; + name = "Projects"; + # page = projects; + } + ]; + + # /notes/* + notes = [ + { + route = [ "notes" "preventing-oom" ]; + name = "Preventing out-of-memory (OOM) errors on Linux"; + page = { cssFile }: markdownToHtml { + name = "preventing-oom"; + markdown = ./notes/preventing-oom.md; + inherit cssFile; + }; + } + { + route = [ "notes" "rust-string-conversions" ]; + name = "Converting between different String types in Rust"; + page = { cssFile }: markdownToHtml { + name = "rust-string-conversions"; + markdown = ./notes/rust-string-conversions.md; + inherit cssFile; + }; + } + ]; + + projects = [ + { + name = "lorri"; + description = "<code>nix-shell</code> replacement for projects"; + link = "https://github.com/nix-community/lorri"; + } + { + name = "netencode"; + description = ''A human-readble nested data exchange format inspired by <a href="https://en.wikipedia.org/wiki/Netstring">netstrings</a> and <a href="https://en.wikipedia.org/wiki/Bencode">bencode</a>.''; + link = depotCgitLink { relativePath = "users/Profpatsch/netencode/README.md"; }; + } + { + name = "yarn2nix"; + description = ''nix dependency generator for the <a href="https://yarnpkg.com/"><code>yarn</code> Javascript package manager</a>''; + link = "https://github.com/Profpatsch/yarn2nix"; + } + ]; + + posts = [ + { + date = "2017-05-04"; + title = "Ligature Emulation in Emacs"; + subtitle = "It’s not pretty, but the results are"; + description = "How to set up ligatures using <code>prettify-symbols-mode</code> and the Hasklig/FiraCode fonts."; + page = { cssFile }: markdownToHtml { + name = "2017-05-04-ligature-emluation-in-emacs"; + markdown = ./posts/2017-05-04-ligature-emulation-in-emacs.md; + inherit cssFile; + }; + route = [ "posts" "2017-05-04-ligature-emluation-in-emacs" ]; + tags = [ "emacs" ]; + } + ]; + + # convert a markdown file to html via lowdown + markdownToHtml = + { name + , # the file to convert + markdown + , # css file to add to the final result, as { route } + cssFile + }: + depot.nix.runExecline "${name}.html" { } ([ + "importas" + "out" + "out" + (depot.users.Profpatsch.lib.debugExec "") + bins.lowdown + "-s" + "-Thtml" + ] ++ + (lib.optional (cssFile != null) ([ "-M" "css=${mkRoute cssFile.route}" ])) + ++ [ + "-o" + "$out" + markdown + ]); + + # takes a { route … } attrset and converts the route lists to an absolute path + fullRoute = attrs: lib.pipe attrs [ + (map (x@{ route, ... }: x // { route = mkRoute route; })) + ]; + + # a cdb from route to a netencoded version of data for each route + router = cssFile: lib.pipe (notes ++ posts) [ + (map (r: with depot.users.Profpatsch.lens; + lib.pipe r [ + (over (field "route") mkRoute) + (over (field "page") (_ { inherit cssFile; })) + ])) + (map (x: { + name = x.route; + value = depot.users.Profpatsch.netencode.gen.dwim x; + })) + lib.listToAttrs + (cdbMake "router") + ]; + + # Create a link to the given source file/directory, given the relative path in the depot repo. + # Checks that the file exists at evaluation time. + depotCgitLink = + { + # relative path from the depot root (without leading /). + relativePath + }: + assert + (lib.assertMsg + (builtins.pathExists (depot.path.origSrc + "/${relativePath}")) + "depotCgitLink: path /${relativePath} does not exist in depot, and depot.path was ${toString depot.path}"); + "https://code.tvl.fyi/tree/${relativePath}"; + + # look up a route by path ($1) + router-lookup = cssFile: depot.nix.writeExecline "router-lookup" { readNArgs = 1; } [ + cdbLookup + (router cssFile) + "$1" + ]; + + runExeclineStdout = name: args: cmd: depot.nix.runExecline name args ([ + "importas" + "-ui" + "out" + "out" + "redirfd" + "-w" + "1" + "$out" + ] ++ cmd); + + notes-index-html = + let o = fullRoute notes; + in '' + <ul> + ${scope o (o: '' + <li><a href="${str o.route}">${esc o.name}</a></li> + '')} + </ul> + ''; + + notes-index = pkgs.writeText "notes-index.html" notes-index-html; + + # A simple mustache-inspired string interpolation combinator + # that takes an object and a template (a function from o to string) + # and returns a string. + scope = o: tpl: + if builtins.typeOf o == "list" then + lib.concatMapStringsSep "\n" tpl o + else if builtins.typeOf o == "set" then + tpl o + else throw "${lib.generators.toPretty {} o} not allowed in template"; + + # string-escape html (TODO) + str = s: s; + # html-escape (TODO) + esc = s: s; + html = s: s; + + projects-index-html = + let o = projects; + in '' + <dl> + ${scope o (o: '' + <dt><a href="${str o.link}">${esc o.name}</a></dt> + <dd>${html o.description}</dd> + '')} + </dl> + ''; + + projects-index = pkgs.writeText "projects-index.html" projects-index-html; + + posts-index-html = + let o = fullRoute posts; + in '' + <dl> + ${scope o (o: '' + <dt>${str o.date} <a href="${str o.route}">${esc o.title}</a></dt> + <dd>${html o.description}</dd> + '')} + </dl> + ''; + + posts-index = pkgs.writeText "projects-index.html" posts-index-html; + + arglibNetencode = val: depot.nix.writeExecline "arglib-netencode" { } [ + "export" + "ARGLIB_NETENCODE" + (depot.users.Profpatsch.netencode.gen.dwim val) + "$@" + ]; + + # A simple http server that serves the site. Yes, it’s horrible. + site-server = { cssFile, port }: depot.nix.writeExecline "blog-server" { } [ + (depot.users.Profpatsch.lib.runInEmptyEnv [ "PATH" ]) + bins.s6-tcpserver + "127.0.0.1" + port + bins.time + "--format=time: %es" + "--" + runOr + return400 + "pipeline" + [ + (arglibNetencode { + what = "request"; + }) + depot.users.Profpatsch.read-http + ] + depot.users.Profpatsch.netencode.record-splice-env + runOr + return500 + "importas" + "-i" + "path" + "path" + "if" + [ depot.tools.eprintf "GET \${path}\n" ] + runOr + return404 + "backtick" + "-ni" + "TEMPLATE_DATA" + [ + # TODO: factor this out of here, this is routing not serving + "ifelse" + [ bins.test "$path" "=" "/notes" ] + [ + "export" + "content-type" + "text/html" + "export" + "serve-file" + notes-index + depot.users.Profpatsch.netencode.env-splice-record + ] + "ifelse" + [ bins.test "$path" "=" "/projects" ] + [ + "export" + "content-type" + "text/html" + "export" + "serve-file" + projects-index + depot.users.Profpatsch.netencode.env-splice-record + ] + "ifelse" + [ bins.test "$path" "=" "/posts" ] + [ + "export" + "content-type" + "text/html" + "export" + "serve-file" + posts-index + depot.users.Profpatsch.netencode.env-splice-record + ] + # TODO: ignore potential query arguments. See 404 message + "pipeline" + [ (router-lookup cssFile) "$path" ] + depot.users.Profpatsch.netencode.record-splice-env + "importas" + "-ui" + "page" + "page" + "export" + "content-type" + "text/html" + "export" + "serve-file" + "$page" + depot.users.Profpatsch.netencode.env-splice-record + ] + runOr + return500 + "if" + [ + "pipeline" + [ + bins.printf + '' + HTTP/1.1 200 OK + Content-Type: {{{content-type}}}; charset=UTF-8 + Connection: close + + '' + ] + depot.users.Profpatsch.netencode.netencode-mustache + ] + "pipeline" + [ "importas" "t" "TEMPLATE_DATA" bins.printf "%s" "$t" ] + depot.users.Profpatsch.netencode.record-splice-env + "importas" + "-ui" + "serve-file" + "serve-file" + bins.cat + "$serve-file" + ]; + + # run argv or $1 if argv returns a failure status code. + runOr = depot.nix.writeExecline "run-or" { readNArgs = 1; } [ + "foreground" + [ "$@" ] + "importas" + "?" + "?" + "ifelse" + [ bins.test "$?" "-eq" "0" ] + [ ] + "if" + [ depot.tools.eprintf "runOr: exited \${?}, running \${1}\n" ] + "$1" + ]; + + return400 = depot.nix.writeExecline "return400" { } [ + bins.printf + "%s" + '' + HTTP/1.1 400 Bad Request + Content-Type: text/plain; charset=UTF-8 + Connection: close + + '' + ]; + + return404 = depot.nix.writeExecline "return404" { } [ + bins.printf + "%s" + '' + HTTP/1.1 404 Not Found + Content-Type: text/plain; charset=UTF-8 + Connection: close + + This page doesn’t exist! Query arguments are not handled at the moment. + '' + ]; + + return500 = depot.nix.writeExecline "return500" { } [ + bins.printf + "%s" + '' + HTTP/1.1 500 Internal Server Error + Content-Type: text/plain; charset=UTF-8 + Connection: close + + Encountered an internal server error. Please try again. + '' + ]; + + capture-stdin = depot.nix.writers.rustSimple + { + name = "capture-stdin"; + dependencies = [ depot.users.Profpatsch.execline.exec-helpers ]; + } '' + extern crate exec_helpers; + use std::io::Read; + fn main() { + let (args, prog) = exec_helpers::args_for_exec("capture-stdin", 1); + let valname = &args[1]; + let mut v : Vec<u8> = vec![]; + std::io::stdin().lock().read_to_end(&mut v).unwrap(); + exec_helpers::exec_into_args("capture-stdin", prog, vec![(valname, v)]); + } + ''; + + # go from a list of path elements to an absolute route string + mkRoute = route: "/" + lib.concatMapStringsSep "/" urlencodeAscii route; + + # urlencodes, but only ASCII characters + # https://en.wikipedia.org/wiki/Percent-encoding + urlencodeAscii = urlPiece: + let + raw = [ "!" "#" "$" "%" "&" "'" "(" ")" "*" "+" "," "/" ":" ";" "=" "?" "@" "[" "]" ]; + enc = [ "%21" "%23" "%24" "%25" "%26" "%27" "%28" "%29" "%2A" "%2B" "%2C" "%2F" "%3A" "%3B" "%3D" "%3F" "%40" "%5B" "%5D" ]; + rest = [ "A" "B" "C" "D" "E" "F" "G" "H" "I" "J" "K" "L" "M" "N" "O" "P" "Q" "R" "S" "T" "U" "V" "W" "X" "Y" "Z" "a" "b" "c" "d" "e" "f" "g" "h" "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t" "u" "v" "w" "x" "y" "z" "0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "-" "_" "." "~" ]; + in + assert lib.assertMsg (lib.all (c: builtins.elem c (raw ++ rest)) (lib.stringToCharacters urlPiece)) + "urlencodeAscii: the urlPiece must only contain valid url ASCII characters, was: ${urlPiece}"; + builtins.replaceStrings raw enc urlPiece; + + + # create a cdb record entry, as required by the cdbmake tool + cdbRecord = key: val: + "+${toString (builtins.stringLength key)},${toString (builtins.stringLength val)}:" + + "${key}->${val}\n"; + + # create a full cdbmake input from an attribute set of keys to values (strings) + cdbRecords = + with depot.nix.yants; + defun [ (attrs (either drv string)) string ] + (attrs: + (lib.concatStrings (lib.mapAttrsToList cdbRecord attrs)) + "\n"); + + # run cdbmake on a list of key/value pairs (strings + cdbMake = name: attrs: depot.nix.runExecline "${name}.cdb" + { + stdin = cdbRecords attrs; + } [ + "importas" + "out" + "out" + depot.users.Profpatsch.lib.eprint-stdin + "if" + [ bins.cdbmake "db" "tmp" ] + bins.mv + "db" + "$out" + ]; + + # look up a key ($2) in the given cdb ($1) + cdbLookup = depot.nix.writeExecline "cdb-lookup" { readNArgs = 2; } [ + # cdb ($1) on stdin + "redirfd" + "-r" + "0" + "$1" + # key ($2) lookup + bins.cdbget + "$2" + ]; + +in +depot.nix.readTree.drvTargets { + inherit + router + depotCgitLink + site-server + notes-index + notes-index-html + projects-index + projects-index-html + posts-index-html + ; + +} diff --git a/users/Profpatsch/blog/notes/preventing-oom.md b/users/Profpatsch/blog/notes/preventing-oom.md new file mode 100644 index 000000000000..59ea4f747700 --- /dev/null +++ b/users/Profpatsch/blog/notes/preventing-oom.md @@ -0,0 +1,33 @@ +tags: linux +date: 2020-01-25 +certainty: likely +status: initial +title: Preventing out-of-memory (OOM) errors on Linux + +# Preventing out-of-memory (OOM) errors on Linux + +I’ve been running out of memory more and more often lately. I don’t use any swap space because I am of the opinion that 16GB of memory should be sufficient for most daily and professional tasks. Which is generally true, however sometimes I have a runaway filling my memory. Emacs is very good at doing this for example, prone to filling your RAM when you open json files with very long lines. + +In theory, the kernel OOM killer should come in and save the day, but the Linux OOM killer is notorious for being extremely … conservative. It will try to free every internal structure it can before even thinking about touching any userspace processes. At that point, the desktop usually stopped responding minutes ago. + +Luckily the kernel provides memory statistics for the whole system, as well as single process, and the [`earlyoom`](https://github.com/rfjakob/earlyoom) tool uses those to keep memory usage under a certain limit. It will start killing processes, “heaviest” first, until the given upper memory limit is satisfied again. + +On NixOS, I set: + +```nix +{ + services.earlyoom = { + enable = true; + freeMemThreshold = 5; # <%5 free + }; +} +``` + +and after activation, this simple test shows whether the daemon is working: + +```shell +$ tail /dev/zero +fish: “tail /dev/zero” terminated by signal SIGTERM (Polite quit request) +``` + +`tail /dev/zero` searches for the last line of the file `/dev/zero`, and since it cannot know that there is no next line and no end to the stream of `\0` this file produces, it will fill the RAM as quickly as physically possible. Before it can fill it completely, `earlyoom` recognizes that the limit was breached, singles out the `tail` command as the process using the most amount of memory, and sends it a `SIGTERM`. diff --git a/users/Profpatsch/blog/notes/rust-string-conversions.md b/users/Profpatsch/blog/notes/rust-string-conversions.md new file mode 100644 index 000000000000..99071ef9d370 --- /dev/null +++ b/users/Profpatsch/blog/notes/rust-string-conversions.md @@ -0,0 +1,53 @@ +# Converting between different String types in Rust + +``` +let s: String = ... +let st: &str = ... +let u: &[u8] = ... +let b: [u8; 3] = b"foo" +let v: Vec<u8> = ... +let os: OsString = ... +let ost: OsStr = ... + +From To Use Comment +---- -- --- ------- +&str -> String String::from(st) +&str -> &[u8] st.as_bytes() +&str -> Vec<u8> st.as_bytes().to_owned() via &[u8] +&str -> &OsStr OsStr::new(st) + +String -> &str &s alt. s.as_str() +String -> &[u8] s.as_bytes() +String -> Vec<u8> s.into_bytes() +String -> OsString OsString::from(s) + +&[u8] -> &str str::from_utf8(u).unwrap() +&[u8] -> String String::from_utf8(u).unwrap() +&[u8] -> Vec<u8> u.to_owned() +&[u8] -> &OsStr OsStr::from_bytes(u) use std::os::unix::ffi::OsStrExt; + +[u8; 3] -> &[u8] &b[..] byte literal +[u8; 3] -> &[u8] "foo".as_bytes() alternative via utf8 literal + +Vec<u8> -> &str str::from_utf8(&v).unwrap() via &[u8] +Vec<u8> -> String String::from_utf8(v) +Vec<u8> -> &[u8] &v +Vec<u8> -> OsString OsString::from_vec(v) use std::os::unix::ffi::OsStringExt; + +&OsStr -> &str ost.to_str().unwrap() +&OsStr -> String ost.to_os_string().into_string() via OsString + .unwrap() +&OsStr -> Cow<str> ost.to_string_lossy() Unicode replacement characters +&OsStr -> OsString ost.to_os_string() +&OsStr -> &[u8] ost.as_bytes() use std::os::unix::ffi::OsStringExt; + +OsString -> String os.into_string().unwrap() returns original OsString on failure +OsString -> &str os.to_str().unwrap() +OsString -> &OsStr os.as_os_str() +OsString -> Vec<u8> os.into_vec() use std::os::unix::ffi::OsStringExt; +``` + + +## Source + +Original source is [this document on Pastebin](https://web.archive.org/web/20190710121935/https://pastebin.com/Mhfc6b9i) diff --git a/users/Profpatsch/blog/posts/2017-05-04-ligature-emulation-in-emacs.md b/users/Profpatsch/blog/posts/2017-05-04-ligature-emulation-in-emacs.md new file mode 100644 index 000000000000..ba80888badd8 --- /dev/null +++ b/users/Profpatsch/blog/posts/2017-05-04-ligature-emulation-in-emacs.md @@ -0,0 +1,123 @@ +title: Ligature Emulation in Emacs +date: 2017-05-04 + +Monday was (yet another) +[NixOS hackathon][hackathon] at [OpenLab Augsburg][ola]. +[Maximilian][mhuber] was there and to my amazement +he got working ligatures in his Haskell files in Emacs! Ever since Hasklig +updated its format to use ligatures and private Unicode code points a while ago, +the hack I had used in my config stopped working. + +Encouraged by that I decided to take a look on Tuesday. Long story short, I was +able to [get it working in a pretty satisfying way][done]. + +[hackathon]: https://www.meetup.com/Munich-NixOS-Meetup/events/239077247/ +[mhuber]: https://github.com/maximilianhuber +[ola]: https://openlab-augsburg.de +[done]: https://github.com/i-tu/Hasklig/issues/84#issuecomment-298803495 + +What’s left to do is package it into a module and push to melpa. + + +### elisp still sucks, but it’s bearable, sometimes + +I’m the kind of person who, when trying to fix something elisp related, normally +gives up two hours later and three macro calls deep. Yes, homoiconic, +non-lexically-scoped, self-rewriting code is not exactly my fetish. +This time the task and the library (`prettify-symbols-mode`) were simple enough +for that to not happen. + +Some interesting technical trivia: + +- elisp literal character syntax is `?c`. `?\t` is the tab character +- You join characters by `(string c1 c2 c3 ...)` +- [dash.el][dash] is pretty awesome and does what a functional programmer + expects. Also, Rainbow Dash. +- Hasklig and FiraCode multi-column symbols actually [only occupy one column, on + the far right of the glyph][glyph]. `my-correct-symbol-bounds` fixes emacs’ + rendering in that case. + + +[dash]: https://github.com/magnars/dash.el +[glyph]: https://github.com/tonsky/FiraCode/issues/211#issuecomment-239082368 + + +## Appendix A + +For reference, here’s the complete code as it stands now. Feel free to paste +into your config; let’s make it [MIT][mit]. Maybe link to this site, in case there are +updates. + +[mit]: https://opensource.org/licenses/MIT + +```elisp + (defun my-correct-symbol-bounds (pretty-alist) + "Prepend a TAB character to each symbol in this alist, +this way compose-region called by prettify-symbols-mode +will use the correct width of the symbols +instead of the width measured by char-width." + (mapcar (lambda (el) + (setcdr el (string ?\t (cdr el))) + el) + pretty-alist)) + + (defun my-ligature-list (ligatures codepoint-start) + "Create an alist of strings to replace with +codepoints starting from codepoint-start." + (let ((codepoints (-iterate '1+ codepoint-start (length ligatures)))) + (-zip-pair ligatures codepoints))) + + ; list can be found at https://github.com/i-tu/Hasklig/blob/master/GlyphOrderAndAliasDB#L1588 + (setq my-hasklig-ligatures + (let* ((ligs '("&&" "***" "*>" "\\\\" "||" "|>" "::" + "==" "===" "==>" "=>" "=<<" "!!" ">>" + ">>=" ">>>" ">>-" ">-" "->" "-<" "-<<" + "<*" "<*>" "<|" "<|>" "<$>" "<>" "<-" + "<<" "<<<" "<+>" ".." "..." "++" "+++" + "/=" ":::" ">=>" "->>" "<=>" "<=<" "<->"))) + (my-correct-symbol-bounds (my-ligature-list ligs #Xe100)))) + + ;; nice glyphs for haskell with hasklig + (defun my-set-hasklig-ligatures () + "Add hasklig ligatures for use with prettify-symbols-mode." + (setq prettify-symbols-alist + (append my-hasklig-ligatures prettify-symbols-alist)) + (prettify-symbols-mode)) + + (add-hook 'haskell-mode-hook 'my-set-hasklig-ligatures) +``` + +## Appendix B (Update 1): FiraCode integration + +I also created a mapping for [FiraCode][fira]. You need to grab the [additional +symbol font][symbol] that adds (most) ligatures to the unicode private use area. +Consult your system documentation on how to add it to your font cache. +Next add `"Fira Code"` and `"Fira Code Symbol"` to your font preferences. Symbol +only contains the additional characters, so you need both. + +If you are on NixOS, the font package should be on the main branch shortly, [I +added a package][symbol-pkg]. + +[fira]: https://github.com/tonsky/FiraCode/ +[symbol]: https://github.com/tonsky/FiraCode/issues/211#issuecomment-239058632 +[symbol-pkg]: https://github.com/NixOS/nixpkgs/pull/25517 + +Here’s the mapping adjusted for FiraCode: + +```elisp + (setq my-fira-code-ligatures + (let* ((ligs '("www" "**" "***" "**/" "*>" "*/" "\\\\" "\\\\\\" + "{-" "[]" "::" ":::" ":=" "!!" "!=" "!==" "-}" + "--" "---" "-->" "->" "->>" "-<" "-<<" "-~" + "#{" "#[" "##" "###" "####" "#(" "#?" "#_" "#_(" + ".-" ".=" ".." "..<" "..." "?=" "??" ";;" "/*" + "/**" "/=" "/==" "/>" "//" "///" "&&" "||" "||=" + "|=" "|>" "^=" "$>" "++" "+++" "+>" "=:=" "==" + "===" "==>" "=>" "=>>" "<=" "=<<" "=/=" ">-" ">=" + ">=>" ">>" ">>-" ">>=" ">>>" "<*" "<*>" "<|" "<|>" + "<$" "<$>" "<!--" "<-" "<--" "<->" "<+" "<+>" "<=" + "<==" "<=>" "<=<" "<>" "<<" "<<-" "<<=" "<<<" "<~" + "<~~" "</" "</>" "~@" "~-" "~=" "~>" "~~" "~~>" "%%" + "x" ":" "+" "+" "*"))) + (my-correct-symbol-bounds (my-ligature-list ligs #Xe100)))) +``` |