diff options
Diffstat (limited to 'users/sterni')
41 files changed, 3401 insertions, 0 deletions
diff --git a/users/sterni/OWNERS b/users/sterni/OWNERS new file mode 100644 index 000000000000..cace4d0f3759 --- /dev/null +++ b/users/sterni/OWNERS @@ -0,0 +1,3 @@ +inherited: false +owners: + - sterni diff --git a/users/sterni/clhs-lookup/README.md b/users/sterni/clhs-lookup/README.md new file mode 100644 index 000000000000..1f42ff43a210 --- /dev/null +++ b/users/sterni/clhs-lookup/README.md @@ -0,0 +1,13 @@ +# clhs-lookup + +Simple cli to lookup symbols' documentation in a local copy of the +Common Lisp HyperSpec. + +## usage + +``` +clhs-lookup [--print] symbol [symbol [...]] + + --print Print documentation paths to stdout instead of + opening them with $BROWSER (defaults to xdg-open). +``` diff --git a/users/sterni/clhs-lookup/clhs-lookup.lisp b/users/sterni/clhs-lookup/clhs-lookup.lisp new file mode 100644 index 000000000000..0e61dd901f93 --- /dev/null +++ b/users/sterni/clhs-lookup/clhs-lookup.lisp @@ -0,0 +1,46 @@ +(in-package :clhs-lookup) +(declaim (optimize (safety 3))) + +(defun find-symbols-paths (syms clhs) + "Find pathnames to HyperSpec files describing the listed + symbol names (as strings). Paths are returned in the order + of the symbols given with missing entries removed." + (check-type syms list) + (check-type clhs pathname) + (let* ((data-dir (merge-pathnames "HyperSpec/Data/" clhs)) + (data (merge-pathnames "Map_Sym.txt" data-dir)) + (found (make-hash-table :test #'equal)) + (syms (mapcar #'string-upcase syms))) + (with-open-file (s data :direction :input) + (loop + with missing = syms + for symbol-line = (read-line s nil :eof) + for path-line = (read-line s nil :eof) + until (or (eq symbol-line :eof) + (eq path-line :eof) + (null missing)) + for pos = (position symbol-line missing :test #'equal) + when pos + do (progn + (delete symbol-line missing) + (setf (gethash symbol-line found) path-line))) + ; TODO(sterni): get rid of Data/../ in path + (mapcar + (lambda (x) (merge-pathnames x data-dir)) + (remove nil + (mapcar (lambda (x) (gethash x found)) syms)))))) + +(defun main () + (let* ((browser (or (uiop:getenvp "BROWSER") "xdg-open")) + (args (uiop:command-line-arguments)) + (prin (member "--print" args :test #'equal)) + (syms (remove-if (lambda (x) (eq (char x 0) #\-)) args)) + (paths (find-symbols-paths syms *clhs-path*))) + (if (null paths) + (uiop:quit 1) + (dolist (p paths) + (if prin + (format t "~A~%" p) + (uiop:launch-program + (format nil "~A ~A" browser p) + :force-shell t)))))) diff --git a/users/sterni/clhs-lookup/default.nix b/users/sterni/clhs-lookup/default.nix new file mode 100644 index 000000000000..b6a0bd06790f --- /dev/null +++ b/users/sterni/clhs-lookup/default.nix @@ -0,0 +1,39 @@ +{ pkgs, depot, ... }: + +let + inherit (pkgs) fetchzip writeText; + inherit (depot.nix) buildLisp; + inherit (builtins) replaceStrings; + + clhsVersion = "7-0"; + + clhs = fetchzip { + name = "HyperSpec-${replaceStrings [ "-" ] [ "." ] clhsVersion}"; + url = "ftp://ftp.lispworks.com/pub/software_tools/reference/HyperSpec-${clhsVersion}.tar.gz"; + sha256 = "1zsi35245m5sfb862ibzy0pzlph48wvlggnqanymhgqkpa1v20ak"; + stripRoot = false; + }; + + clhs-path = writeText "clhs-path.lisp" '' + (in-package :clhs-lookup.clhs-path) + (defparameter *clhs-path* (pathname "${clhs}/")) + ''; + + clhs-lookup = buildLisp.program { + name = "clhs-lookup"; + + deps = [ + { + default = buildLisp.bundled "asdf"; + sbcl = buildLisp.bundled "uiop"; + } + ]; + + srcs = [ + ./packages.lisp + clhs-path + ./clhs-lookup.lisp + ]; + }; +in + clhs-lookup diff --git a/users/sterni/clhs-lookup/packages.lisp b/users/sterni/clhs-lookup/packages.lisp new file mode 100644 index 000000000000..d059b96ce9f0 --- /dev/null +++ b/users/sterni/clhs-lookup/packages.lisp @@ -0,0 +1,10 @@ +(defpackage :clhs-lookup.clhs-path + (:use :cl) + (:export :*clhs-path*)) + +(defpackage clhs-lookup + (:use :cl :uiop) + (:import-from :clhs-lookup.clhs-path :*clhs-path*) + (:export :main + :find-symbols-paths)) + diff --git a/users/sterni/dot-time-man-pages/OWNERS b/users/sterni/dot-time-man-pages/OWNERS new file mode 100644 index 000000000000..980c17b424f2 --- /dev/null +++ b/users/sterni/dot-time-man-pages/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - edef diff --git a/users/sterni/dot-time-man-pages/default.nix b/users/sterni/dot-time-man-pages/default.nix new file mode 100644 index 000000000000..bf7d63dbd797 --- /dev/null +++ b/users/sterni/dot-time-man-pages/default.nix @@ -0,0 +1,70 @@ +{ depot, lib, ... }: + +let + # TODO(sterni): find a better place for this: is dot time //fun? + + # get the email address of a depot user from //ops/users + findEmail = user: + let + res = builtins.filter ({ username, ... }: username == user) depot.ops.users; + len = builtins.length res; + in + if len == 1 + then (builtins.head res).email + else builtins.throw "findEmail: got ${toString len} results instead of 1"; + + # dot-time(7) man page, ported from dotti.me + dot-time = rec { + name = "dot-time"; + section = 7; + content = '' + .Dd $Mdocdate$ + .Dt ${lib.toUpper name} ${toString section} + .Os + .Sh NAME + .Nm ${name} + .Nd a universal convention for conveying time + .Sh DESCRIPTION + For those of us who travel often or coordinate across many timezones, + working with local time is frequently impractical. + ISO8601, in all its wisdom, allows for time zone designators, + but still represents the hours and minutes as local time, + thus making it inconvenient for quickly comparing timestamps from + different locations. + .Pp + Dot time instead uses UTC for all date, hour, and minute indications, + and while it allows for time zone designators, they are optional + information that can be dropped without changing the indicated time. + It uses an alternate hour separator to make it easy to distinguish from + regular ISO8601. + When a time zone designator is provided, one can easily obtain + the matching local time by adding the UTC offset to the UTC time. + .Sh EXAMPLES + These timestamps all represent the same point in time. + .TS + allbox tab(|); + lb | lb | lb + l | l | l. + dot time|ISO8601|RFC3339 + 2019-06-19T22·13-04|2019-06-19T18:13-04|2019-06-19T18:13:00-04:00 + 2019-06-19T22·13+00|2019-06-19T22:13+00|2019-06-19T22:13:00Z + 2019-06-19T22·13+02|2019-06-20T00:13+02|2019-06-20T00:13:00+02:00 + .TE + .Sh SEE ALSO + .Lk https://dotti.me dotti.me + .Sh AUTHORS + .An -nosplit + .Sy dot time + has been proposed and documented by + .An edef Aq Mt ${findEmail "edef"} + and ported to + .Xr mdoc 7 + by + .An sterni Aq Mt ${findEmail "sterni"} . + ''; + }; + +in + depot.nix.buildManPages "dot-time" {} [ + dot-time + ] diff --git a/users/sterni/emacs/default.nix b/users/sterni/emacs/default.nix new file mode 100644 index 000000000000..f7bdf21a025e --- /dev/null +++ b/users/sterni/emacs/default.nix @@ -0,0 +1,43 @@ +{ depot, pkgs, ... }: + +let + inherit (pkgs.emacsGcc.pkgs) withPackages; + + emacs = withPackages (epkgs: [ + # basic setup + epkgs.elpaPackages.undo-tree + epkgs.melpaPackages.evil + epkgs.melpaPackages.evil-collection + epkgs.melpaPackages.use-package + # languages + epkgs.bqn-mode + epkgs.elpaPackages.ada-mode + epkgs.melpaPackages.adoc-mode + epkgs.melpaPackages.dockerfile-mode + epkgs.melpaPackages.haskell-mode + epkgs.melpaPackages.jq-mode + epkgs.melpaPackages.markdown-mode + epkgs.melpaPackages.nix-mode + epkgs.melpaPackages.sly + epkgs.melpaPackages.yaml-mode + epkgs.rust-mode + epkgs.urweb-mode + # misc + epkgs.melpaPackages.hl-todo + epkgs.elpaPackages.rainbow-mode + epkgs.melpaPackages.rainbow-delimiters + # beyond text editing + epkgs.melpaPackages.elfeed + epkgs.melpaPackages.magit + epkgs.tvlPackages.tvl + ]); +in + +# sadly we can't give an init-file via the command line +pkgs.writeShellScriptBin "emacs" '' + exec ${emacs}/bin/emacs \ + --no-init-file \ + --directory ${./.} \ + --eval "(require 'init)" \ + "$@" +'' diff --git a/users/sterni/emacs/init.el b/users/sterni/emacs/init.el new file mode 100644 index 000000000000..4b868cb242d1 --- /dev/null +++ b/users/sterni/emacs/init.el @@ -0,0 +1,184 @@ +;; Set default font and fallback font via set-fontset-font +;; TODO(sterni): Investigate non-emoji representation of some glyphs +(let ((mono-font "Bitstream Vera Sans Mono-12") + (emoji-font "Noto Color Emoji-12")) + (setq default-frame-alist `((font . ,mono-font))) + (set-frame-font mono-font t t) + (set-fontset-font t nil emoji-font)) + +(setq inhibit-startup-message t + display-time-24hr-format t + select-enable-clipboard t) + +;; Reload files +(global-auto-revert-mode 1) + +;; Indent +(set-default 'indent-tabs-mode nil) +(setq tab-width 2) + +;; UTF-8 +(setq locale-coding-system 'utf-8) +(set-terminal-coding-system 'utf-8) +(set-keyboard-coding-system 'utf-8) +(set-selection-coding-system 'utf-8) +(prefer-coding-system 'utf-8) + +;; Disable unnecessary GUI elements +(scroll-bar-mode 0) +(menu-bar-mode 0) +(tool-bar-mode 0) + +(add-hook 'after-make-frame-functions + (lambda (frame) (scroll-bar-mode 0))) + +;; don't center on cursor when scrolling +(setq scroll-conservatively 1) + +;; type less +(defalias 'yes-or-no-p 'y-or-n-p) + +;; Extra settings when graphical session +(when window-system + (setq frame-title-format '(buffer-file-name "%f" ("%b"))) + (mouse-wheel-mode t) + (blink-cursor-mode -1)) + +;; TODO(sterni): prevent some remaining backup files +(setq auto-save-file-name-transforms + `((".*" ,temporary-file-directory t))) +(setq backup-directory-alist + `((".*" . ,temporary-file-directory))) + +;; buffers +;; unique component should come first for better completion +(setq uniquify-buffer-name-style 'forward) + +;; Display column numbers +(column-number-mode t) +(setq-default fill-column 80) +(setq display-fill-column-indicator-column t) +(add-hook 'prog-mode-hook #'display-fill-column-indicator-mode) + +;; whitespace +(setq whitespace-style '(face trailing tabs) + whitespace-line-column fill-column) +(add-hook 'prog-mode-hook #'whitespace-mode) + +;;; Configure built in modes + +;; Perl +(setq perl-indent-level 2) +(setq perl-continued-statement-offset 0) +(setq perl-continued-brace-offset 0) + +;;; Configure packages +(require 'use-package) + +(package-initialize) + +(use-package undo-tree + :config + (global-undo-tree-mode)) + +(use-package magit + :after evil + :config + ; reset (buffer-local) fill-column value to emacs' default + ; gerrit doesn't like 80 column commit messages… + (add-hook 'git-commit-mode-hook (lambda () (setq fill-column 72))) + (evil-define-key 'normal 'global (kbd "<leader>gr") 'magit-status)) +(use-package tvl :after magit) + +(setq ediff-split-window-function 'split-window-horizontally) + +(use-package evil + :init + (setq evil-want-integration t) + (setq evil-want-keybinding nil) + (setq evil-shift-width 2) + (setq evil-split-window-below t) + (setq evil-split-window-right t) + (setq evil-undo-system 'undo-tree) + :config + (evil-mode 1) + (evil-set-leader 'normal ",") ;; TODO(sterni): space would be nice, but… + (evil-set-leader 'visual ",") + ;; buffer management + (evil-define-key 'normal 'global (kbd "<leader>bk") 'kill-buffer) + (evil-define-key 'normal 'global (kbd "<leader>bb") 'switch-to-buffer) + ;; window management + (evil-define-key 'normal 'global (kbd "<leader>wk") 'delete-window) + (evil-define-key 'normal 'global (kbd "<leader>wo") 'delete-other-window) + (evil-define-key 'normal 'global (kbd "<leader>wh") 'split-window-below) + (evil-define-key 'normal 'global (kbd "<leader>wv") 'split-window-right) + (evil-define-key 'normal 'global (kbd "<leader>ww") 'other-window) + ;; emacs + (evil-define-key 'visual 'global (kbd "<leader>ee") 'eval-region) + (evil-define-key 'normal 'global (kbd "<leader>ee") 'eval-last-sexp) + (evil-define-key 'normal 'global (kbd "<leader>ep") 'eval-print-last-sexp) + (evil-define-key 'normal 'global (kbd "<leader>eh") 'help) + ;; modify what is displayed + (evil-define-key 'normal 'global (kbd "<leader>dw") + (lambda () + (interactive) + (whitespace-mode 'toggle) + (display-fill-column-indicator-mode 'toggle))) + ;; elfeed bindings for evil (can't use-package elfeed apparently) + (evil-define-key 'normal 'global (kbd "<leader>ff") 'elfeed) + (evil-define-key '(normal visual) elfeed-search-mode-map + (kbd "o") 'elfeed-search-browse-url + (kbd "r") 'elfeed-search-untag-all-unread + (kbd "u") 'elfeed-search-tag-all-unread + (kbd "<leader>ff") 'elfeed-search-fetch + (kbd "<leader>fc") 'elfeed-db-compact + (kbd "<leader>fr") 'elfeed-search-update--force)) + +(use-package evil-collection + :after evil + :config + (evil-collection-init)) + +(use-package rainbow-delimiters + :hook (prog-mode . rainbow-delimiters-mode)) + +(use-package nix-mode :mode "\\.nix\\'") +(use-package nix-drv-mode :mode "\\.drv\\'") + +(use-package haskell-mode) +(use-package urweb-mode) +(use-package bqn-mode + :mode "\\.bqn\\'" + :custom bqn-mode-map-prefix "C-s-") ; probably rather using C-\ +(use-package yaml-mode) +(use-package dockerfile-mode) +(use-package jq-mode + :config (add-to-list 'auto-mode-alist '("\\.jq\\'" . jq-mode))) +(use-package rust-mode) +(use-package sly + :after evil + :hook ((sly-mrepl-mode . (lambda () (rainbow-delimiters-mode-enable)))) + :config + (evil-define-key 'normal sly-mrepl-mode-map (kbd "C-r") 'isearch-backward)) + +(use-package ada-mode) + +(use-package rainbow-mode) +(use-package hl-todo + :hook ((prog-mode . hl-todo-mode)) + :config + (setq hl-todo-keyword-faces + '(("TODO" . "#FF0000") + ("FIXME" . "#FF0000") + ("HACK" . "#7f7f7f") + ("XXX" . "#aa0000")))) + +(use-package markdown-mode + :commands (markdown-mode gfm-mode) + :mode (("\\.md\\'" . markdown-mode))) +(use-package adoc-mode + :mode (("\\.adoc\\'" . adoc-mode))) + +(require 'subscriptions) + +(provide 'init) diff --git a/users/sterni/emacs/subscriptions.el b/users/sterni/emacs/subscriptions.el new file mode 100644 index 000000000000..bf890a5ab8b3 --- /dev/null +++ b/users/sterni/emacs/subscriptions.el @@ -0,0 +1,88 @@ +;;; elfeed subscriptions + +(setq elfeed-feeds + (append + ;; immutable subscriptions tracked in git + '(("https://repology.org/maintainer/sternenseemann%40systemli.org/feed-for-repo/nix_unstable/atom" dashboard releases) + ("http://hundimbuero.blogspot.com/feeds/posts/default?alt=rss" blog cool-and-nice) + ("gopher://text.causal.agency/0feed.atom" blog) + ("http://xsteadfastx.org/feed/" blog cool-and-nice) + ("https://tvl.fyi/feed.atom" blog cool-and-nice) + ("https://hannes.robur.coop/atom" blog) + ("https://stevelosh.com/rss.xml" blog) + ("https://planet.lisp.org/rss20.xml" blog) + ("https://hyperthings.garden/rss/all-posts.xml" blog) + ("https://blog.benjojo.co.uk/rss.xml" blog) + ("https://leahneukirchen.org/blog/index.atom" blog cool-and-nice) + ("https://leahneukirchen.org/trivium/index.atom" blog links cool-and-nice) + ("https://firefly.nu/feeds/all.atom.xml" blog cool-and-nice) + ("https://tazj.in/feed.atom" blog cool-and-nice) + ("https://alyssa.is/feed.xml" blog cool-and-nice) + ("https://eta.st/feed.xml" blog cool-and-nice) + ("https://h.eta.st/rss" noisy cool-and-nice) + ("https://spectrum-os.org/git/www/atom/bibliography.html" links blog) + ("https://rachelbythebay.com/w/atom.xml" blog) + ("http://evrl.com/feed.xml" blog) + ("https://vulns.xyz/feed.xml" blog) + ("https://www.german-foreign-policy.com/?type=9818" news) + ("https://niedzejkob.p4.team/rss.xml" blog) + ("https://grahamc.com/feed/" blog) + ("https://michael.stapelberg.ch/feed.xml" blog) + ("https://kazu-yamamoto.hatenablog.jp/feed" blog) + ("https://ariadne.space/feed/" blog) + ("https://bodil.lol/rss.xml" blog) + ("http://blog.nullspace.io/feed.xml" blog) + ("https://blog.kingcons.io/rss.xml" blog) + ("http://jaspervdj.be/rss.xml" blog) + ("https://christine.website/blog.rss" blog) + ("https://drewdevault.com/feed.xml" blog) + ("https://www.imperialviolet.org/iv-rss.xml" blog) + ("https://latacora.micro.blog/feed.xml" blog) + ("https://22gato.tumblr.com/rss" pictures cool-and-nice) + ("https://theprofoundprogrammer.com/rss" blog) + ("https://wiki.openlab-augsburg.de/_feed" openlab) + ("http://shitopenlabsays.tumblr.com/rss" openlab) + ("http://suckless.org/atom.xml" releases) + ("https://kristaps.bsd.lv/lowdown/atom.xml" releases) + ("https://www.tweag.io/rss.xml" blog) + ("http://planet.haskell.org/atom.xml" planet blog) + ("http://0pointer.net/blog/index.atom" blog) + ("https://emacsninja.com/feed.atom" blog) + ("https://emacshorrors.com/feed.atom" blog) + ("http://therealmntmn.tumblr.com/rss" blog) + ("http://blog.duangle.com/feeds/posts/default" blog) + ("http://blog.johl.io/atom.xml" blog) + ("http://blog.z3bra.org/rss/feed.xml" blog) + ("http://ccc.de/de/rss/updates.xml" news) + ;; ("http://fabienne.us/feed/" blog) ; database error + ("http://feeds.feedburner.com/baschtcom" blog) + ("http://ffaaaffaffaffaa.tumblr.com/rss" pictures) + ("http://fnordig.de/feed.xml" blog) + ("http://fotografiona.tumblr.com/rss" pictures) + ("https://grandhotel-cosmopolis.org/de/feed" news) + ("http://guteaussicht.org/rss" pictures) + ("http://konvergenzfehler.de/feed/" blog) + ("https://markuscisler.com/feed.xml" blog) + ("http://n00bcore.de/feed/" podcast) + ("http://spacethatneverwas.tumblr.com/rss" pictures) + ("http://theresa.someserver.de/blog/?feed=rss2" blog) + ("http://www.frumble.de/blog/feed/" blog) + ("http://www.plomlompom.de/PlomRogue/plomwiki.php?action=Blog_Atom" blog) + ("http://www.whvrt.de/rss" pictures) + ("http://www.windytan.com/feeds/posts/default" blog) + ("https://echtsuppe.wordpress.com/feed/" blog defunct) + ("https://mgsloan.com/feed.xml" blog) + ("https://notes.sterni.lv/atom.xml" me) + ("http://arduina.net/feed/" defunct blog) + ("https://anchor.fm/s/94bb000/podcast/rss" podcast)) + ;; http://www.wollenzin.de/feed/ ;_; + + ;; add more feeds from an untracked file in $HOME + (let ((file (concat (getenv "HOME") + "/.config/emacs-custom/mutable-subscriptions.el"))) + (when (file-exists-p file) + (read (with-temp-buffer + (insert-file-contents file) + (buffer-string))))))) + +(provide 'subscriptions) diff --git a/users/sterni/exercises/aoc/.gitignore b/users/sterni/exercises/aoc/.gitignore new file mode 100644 index 000000000000..de53cfc531cb --- /dev/null +++ b/users/sterni/exercises/aoc/.gitignore @@ -0,0 +1 @@ +/*/input \ No newline at end of file diff --git a/users/sterni/exercises/aoc/2021/solutions.bqn b/users/sterni/exercises/aoc/2021/solutions.bqn new file mode 100755 index 000000000000..7aac53b9eaa7 --- /dev/null +++ b/users/sterni/exercises/aoc/2021/solutions.bqn @@ -0,0 +1,188 @@ +#!/usr/bin/env BQN + +# +# Utilities +# + +IsAsciiNum ← ('0'⊸≤∧≤⟜'9') + +ReadInt ← {(𝕨⊸×+⊣)´∘⌽-⟜'0'𝕩} # stolen from leah2 +ReadDec ← 10⊸ReadInt + +ReadInput ← {•file.Lines ∾ •path‿"/input/day"‿(•Fmt 𝕩)} + +SplitOn ← ((⊢ (-1˙)⍟⊣¨ +`)∘=⊔⊢) + +_fix ← {𝕩 𝕊∘⊢⍟≢ 𝔽 𝕩} + +# +# 2021-12-01 +# + +# part 1 + +day1ExampleData ← 199‿200‿208‿210‿200‿207‿240‿269‿260‿263 +day1Input ← ReadDec¨ReadInput 1 + +# NB: Because distance from the ground is never smaller than zero, it's +# no problem that nudge inserts a zero at the end of the right list +PositiveDeltaCount ← +´∘(⊢<«)+˝˘∘↕ + +! 7 = 1 PositiveDeltaCount day1ExampleData + +•Out "Day 1.1: "∾•Fmt 1 PositiveDeltaCount day1Input + +# part 2 + +! 5 = 3 PositiveDeltaCount day1ExampleData + +•Out "Day 1.2: "∾•Fmt 3 PositiveDeltaCount day1Input + +# +# 2021-12-02 +# + +# part 1 + +day2ExampleData ← ⟨ + "forward 5", + "down 5", + "forward 8", + "up 3", + "down 8", + "forward 2", +⟩ + +day2Input ← ReadInput 2 + +ParseSubmarineCommand ← (((↕2)⊸((((-1)⊸⋆)∘(2⊸|))×(=⟜(⌊∘(÷⟜2))))∘("duf"⊸⊐)∘⊑)×ReadDec∘(IsAsciiNum/⊢)) + +SubmarineDestProduct ← {×´+´ParseSubmarineCommand¨𝕩} + +! 150 = SubmarineDestProduct day2ExampleData + +•Out "Day 2.1: "∾•Fmt SubmarineDestProduct day2Input + +# part 2 + +SubmarineAimedDestProduct ← { + ×´+´((×´)∘(1⊸↓)≍(1⊸⊑))¨ (<0‿0‿0) (⊢∾((⊑∘⌽⊣)+(⊑⊢)))` ParseSubmarineCommand¨𝕩 +} + +! 900 = SubmarineAimedDestProduct day2ExampleData + +•Out "Day 2.2: "∾•Fmt SubmarineAimedDestProduct day2Input + +# +# 2021-12-03 +# + +BinTable ← '0'-˜> + +day3ExampleData ← BinTable ⟨ + "00100", + "11110", + "10110", + "10111", + "10101", + "01111", + "00111", + "11100", + "10000", + "11001", + "00010", + "01010", +⟩ + +day3Input ← BinTable ReadInput 3 + +DeBinList ← ((2⊸×)+⊣)´⌽ +_tableAggr ← {((÷⟜2)∘(/⟜⥊)´∘⌽∘≢𝔽(+˝))𝕩} +GammaRate ← < _tableAggr + +! 22 = DeBinList GammaRate day3ExampleData +! 9 = DeBinList ¬GammaRate day3ExampleData + +•Out "Day 3.1: "∾•Fmt (¬×○DeBinList⊢) GammaRate day3Input + +_lifeSupportRating ← { + # Need to rename the arguments, otherwise the ternary expr becomes a function + bitPos ← 𝕨 + Cmp ← 𝔽 + + crit ← Cmp _tableAggr 𝕩 + matchPos ← bitPos ⊑˘ crit ((⥊˜⟜≢)=⊢) 𝕩 + match ← matchPos/𝕩 + {1=≠match?⊏match;(bitPos+1) Cmp _lifeSupportRating match} +} + +OxygenGeneratorRating ← DeBinList 0 ≤_lifeSupportRating ⊢ +CO2ScrubberRating ← DebinList 0 >_lifeSupportRating ⊢ + +! 23 = OxygenGeneratorRating day3ExampleData +! 10 = CO2ScrubberRating day3ExampleData + +•Out "Day 3.2: "∾•Fmt (OxygenGeneratorRating×CO2ScrubberRating) day3Input + +# +# 2021-12-07 +# + +# part 1 + +day6ExampleData ← ⟨16,1,2,0,4,2,7,1,2,14⟩ +day6Input ← ReadDec¨ ',' SplitOn ⊑ReadInput 6 + +PossiblePositions ← (⌊´+⟜(↕1⊸+)⌈´) +FuelConsumption ← +˝∘|∘(-⌜) +_lowestFuelPossible ← {⌊´∘(𝔽⟜PossiblePositions)˜ 𝕩} + +! 37 = FuelConsumption _lowestFuelPossible day6ExampleData + +•Out "Day 7.1: "∾•Fmt FuelConsumption _lowestFuelPossible day6Input + +# part 2 + +TriNum ← 1⊸+×÷⟜2 + +FuelConsumption2 ← +˝∘(TriNum¨)∘|∘(-⌜) + +! 168 = FuelConsumption2 _lowestFuelPossible day6ExampleData + +•Out "Day 7.2: "∾•Fmt FuelConsumption2 _lowestFuelPossible day6Input + +# +# 2021-12-09 +# + +# part 1 + +ParseHeightMap ← ((≠≍(≠⊑))⥊∾)∘-⟜'0' + +day9ExampleData ← ParseHeightMap ⟨ + "2199943210", + "3987894921", + "9856789892", + "8767896789", + "9899965678" +⟩ +day9Input ← ParseHeightMap ReadInput 9 + +Rotate ← (⍉⌽)∘⊢⍟⊣ # counter clockwise +LowPoints ← {∧´𝕩⊸(⊣<((-⊢) Rotate ∞⊸»˘∘Rotate˜))¨ ↕4} + +RiskLevelSum ← (+´⥊)∘(1⊸+×LowPoints) + +! 15 = RiskLevelSum day9ExampleData + +•Out "Day 9.1: "∾•Fmt RiskLevelSum day9Input + +# part 2 + +NumberBasins ← ((1⊸+⊒⌾⥊)×⊢)∘LowPoints +Basins ← {𝕩⊸((<⟜9⊣)∧(«⌈»⌈«˘⌈»˘⌈⊢)∘⊢) _fix NumberBasins 𝕩} +LargestBasinsProduct ← {×´ 3↑ ∨ 1↓ ≠¨ ⊔⥊Basins 𝕩} + +! 1134 = LargestBasinsProduct day9ExampleData + +•Out "Day 9.2: "∾•Fmt LargestBasinsProduct day9Input diff --git a/users/sterni/htmlman/README.md b/users/sterni/htmlman/README.md new file mode 100644 index 000000000000..258233d4c4d2 --- /dev/null +++ b/users/sterni/htmlman/README.md @@ -0,0 +1,36 @@ +# htmlman + +static site generator for man pages intended for +rendering man page documentation viewable using +a web browser. + +## usage + +If you have a nix expression, `doc.nix`, like this: + +```nix +{ depot, ... }: + +depot.users.sterni.htmlman { + title = "foo project"; + pages = [ + { + name = "foo"; + section = 1; + } + { + name = "foo"; + section = 3; + path = ../devman/foo.3; + } + ]; + manDir = ../man; +} +``` + +You can run the following to directly deploy the resulting +documentation output to a specific target directory: + +```sh +nix-build -A deploy doc.nix && ./result target_directory +``` diff --git a/users/sterni/htmlman/default.nix b/users/sterni/htmlman/default.nix new file mode 100644 index 000000000000..b88bc264103b --- /dev/null +++ b/users/sterni/htmlman/default.nix @@ -0,0 +1,234 @@ +{ depot, lib, pkgs, ... }: + +let + inherit (depot.nix) + getBins + runExecline + yants + ; + + inherit (depot.tools) + cheddar + ; + + inherit (pkgs) + mandoc + coreutils + fetchurl + writers + ; + + bins = getBins cheddar [ "cheddar" ] + // getBins mandoc [ "mandoc" ] + // getBins coreutils [ "cat" "mv" "mkdir" ] + ; + + normalizeDrv = fetchurl { + url = "https://necolas.github.io/normalize.css/8.0.1/normalize.css"; + sha256 = "04jmvybwh2ks4dlnfa70sb3a3z3ig4cv0ya9rizjvm140xq1h22q"; + }; + + execlineStdoutInto = target: line: [ + "redirfd" "-w" "1" target + ] ++ line; + + # I will not write a pure nix markdown renderer + # I will not write a pure nix markdown renderer + # I will not write a pure nix markdown renderer + # I will not write a pure nix markdown renderer + # I will not write a pure nix markdown renderer + markdown = md: + let + html = runExecline.local "rendered-markdown" { + stdin = md; + } ([ + "importas" "-iu" "out" "out" + ] ++ execlineStdoutInto "$out" [ + bins.cheddar "--about-filter" "description.md" + ]); + in builtins.readFile html; + + indexTemplate = { title, description, pages ? [] }: '' + <!doctype html> + <html> + <head> + <meta charset="utf-8"> + <title>${title}</title> + <link rel="stylesheet" type="text/css" href="style.css"/> + </head> + <body> + <div class="index-text"> + <h1>${title}</h1> + ${markdown description} + <h2>man pages</h2> + <ul> + ${lib.concatMapStrings ({ name, section, ... }: '' + <li><a href="${name}.${toString section}.html">${name}(${toString section})</a></li> + '') pages} + </ul> + </div> + </body> + </html> + ''; + + defaultStyle = import ./defaultStyle.nix { }; + + # This deploy script automatically copies the build result into + # a TARGET directory and marks it as writeable optionally. + # It is exposed as the deploy attribute of the result of + # htmlman, so an htmlman expression can be used like this: + # nix-build -A deploy htmlman.nix && ./result target_dir + deployScript = title: drv: writers.writeDash "deploy-${title}" '' + usage() { + printf 'Usage: %s [-w] TARGET\n\n' "$0" + printf 'Deploy htmlman documentation to TARGET directory.\n\n' + printf ' -h Display this help message\n' + printf ' -w Make TARGET directory writeable\n' + } + + if test "$#" -lt 1; then + usage + exit 100 + fi + + writeable=false + + while test "$#" -gt 0; do + case "$1" in + -h) + usage + exit 0 + ;; + -w) + writeable=true + ;; + -*) + usage + exit 100 + ;; + *) + if test -z "$target"; then + target="$1" + else + echo "Too many arguments" + exit 100 + fi + ;; + esac + + shift + done + + if test -z "$target"; then + echo "Missing TARGET" + usage + exit 100 + fi + + set -ex + + mkdir -p "$target" + cp -RTL --reflink=auto "${drv}" "$target" + + if $writeable; then + chmod -R +w "$target" + fi + ''; + + htmlman = + { title + # title of the index page + , description ? "" + # description which is displayed after + # the main heading on the index page + , pages ? [] + # man pages of the following structure: + # { + # name : string; + # section : int; + # path : either path string; + # } + # path is optional, if it is not given, + # the man page source must be located at + # "${manDir}/${name}.${toString section}" + , manDir ? null + # directory in which man page sources are located + , style ? defaultStyle + # CSS to use as a string + , normalizeCss ? true + # whether to include normalize.css before the custom CSS + , linkXr ? "all" + # How to handle cross references in the html output: + # + # * none: don't convert cross references into hyperlinks + # * all: link all cross references as if they were + # rendered into $out by htmlman + # * inManDir: link to all man pages which have their source + # in `manDir` and use the format string defined + # in linkXrFallback for all other cross references. + , linkXrFallback ? "https://manpages.debian.org/unstable/%N.%S.en.html" + # fallback link to use if linkXr == "inManDir" and the man + # page is not in ${manDir}. Placeholders %N (name of page) + # and %S (section of page) can be used. See mandoc(1) for + # more information. + }: + + let + linkXrEnum = yants.enum "linkXr" [ "all" "inManDir" "none" ]; + + index = indexTemplate { + inherit title description pages; + }; + + resolvePath = { path ? null, name, section }: + if path != null + then path + else "${manDir}/${name}.${toString section}"; + + mandocOpts = lib.concatStringsSep "," ([ + "style=style.css" + ] ++ linkXrEnum.match linkXr { + all = [ "man=./%N.%S.html" ]; + inManDir = [ "man=./%N.%S.html;${linkXrFallback}" ]; + none = [ ]; + }); + + html = + runExecline.local "htmlman-${title}" { + derivationArgs = { + inherit index style; + passAsFile = [ "index" "style" ]; + }; + } ([ + "multisubstitute" [ + "importas" "-iu" "out" "out" + "importas" "-iu" "index" "indexPath" + "importas" "-iu" "style" "stylePath" + ] + "if" [ bins.mkdir "-p" "$out" ] + "if" [ bins.mv "$index" "\${out}/index.html" ] + "if" (execlineStdoutInto "\${out}/style.css" [ + "if" ([ + bins.cat + ] ++ lib.optional normalizeCss normalizeDrv + ++ [ + "$style" + ]) + ]) + # let mandoc check for available man pages + "execline-cd" "${manDir}" + ] ++ lib.concatMap ({ name, section, ... }@p: + execlineStdoutInto "\${out}/${name}.${toString section}.html" [ + "if" [ + bins.mandoc + "-mdoc" + "-T" "html" + "-O" mandocOpts + (resolvePath p) + ] + ]) pages); + in html // { + deploy = deployScript title html; + }; +in + htmlman diff --git a/users/sterni/htmlman/defaultStyle.nix b/users/sterni/htmlman/defaultStyle.nix new file mode 100644 index 000000000000..a44b5ef06934 --- /dev/null +++ b/users/sterni/htmlman/defaultStyle.nix @@ -0,0 +1,49 @@ +{ ... }: + +'' + body { + font-size: 1em; + line-height: 1.5; + font-family: serif; + background-color: #efefef; + } + + h1, h2, h3, h4, h5, h6 { + font-family: sans-serif; + font-size: 1em; + margin: 5px 0; + } + + h1 { + margin-top: 0; + } + + a:link, a:visited { + color: #3e7eff; + } + + h1 a, h2 a, h3 a, h4 a, h5 a, h6 a { + text-decoration: none; + } + + .manual-text, .index-text { + padding: 20px; + max-width: 800px; + background-color: white; + margin: 0 auto; + } + + table.head, table.foot { + display: none; + } + + .Nd { + display: inline; + } + + /* use same as cheddar for man pages */ + pre { + padding: 16px; + background-color: #f6f8fa; + } +'' diff --git a/users/sterni/keys.nix b/users/sterni/keys.nix new file mode 100644 index 000000000000..815f62ee080e --- /dev/null +++ b/users/sterni/keys.nix @@ -0,0 +1,7 @@ +{ ... }: + +{ + all = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJk+KvgvI2oJTppMASNUfMcMkA2G5ZNt+HnWDzaXKLlo lukas@wolfgang" + ]; +} diff --git a/users/sterni/mblog/cli.lisp b/users/sterni/mblog/cli.lisp new file mode 100644 index 000000000000..93be7e8b8e44 --- /dev/null +++ b/users/sterni/mblog/cli.lisp @@ -0,0 +1,17 @@ +(in-package :mblog) +(declaim (optimize (safety 3))) + +(defparameter +synopsis+ "mnote-html FILE [FILE [ ... ]]") + +;; TODO(sterni): handle relevant conditions +(defun main () + (let* ((args (uiop:command-line-arguments)) + (help-p (or (not args) + (find-if (lambda (x) + (member x '("-h" "--help" "--usage") + :test #'string=)) + args)))) + (if help-p (format *error-output* "Usage: ~A~%" +synopsis+) + (loop for arg in args + do (apple-note-html-fragment + (mime:mime-message (pathname arg)) *standard-output*))))) diff --git a/users/sterni/mblog/default.nix b/users/sterni/mblog/default.nix new file mode 100644 index 000000000000..16ae573ba78c --- /dev/null +++ b/users/sterni/mblog/default.nix @@ -0,0 +1,31 @@ +{ depot, pkgs, ... }: + +depot.nix.buildLisp.program { + name = "mnote-html"; + + srcs = [ + ./packages.lisp + ./transformer.lisp + ./note.lisp + ./cli.lisp + ]; + + deps = [ + { + sbcl = depot.nix.buildLisp.bundled "uiop"; + default = depot.nix.buildLisp.bundled "asdf"; + } + depot.third_party.lisp.alexandria + depot.third_party.lisp.closure-html + depot.third_party.lisp.cl-who + depot.third_party.lisp.mime4cl + ]; + + main = "mblog:main"; + + # due to sclf + brokenOn = [ + "ccl" + "ecl" + ]; +} diff --git a/users/sterni/mblog/note.lisp b/users/sterni/mblog/note.lisp new file mode 100644 index 000000000000..fa4de0956ffb --- /dev/null +++ b/users/sterni/mblog/note.lisp @@ -0,0 +1,60 @@ +(in-package :mblog) +(declaim (optimize (safety 3))) + +;;; util + +(defun html-escape-stream (in out) + "Escape characters read from stream IN and write them to + stream OUT escaped using WHO:ESCAPE-CHAR-MINIMAL." + (loop for char = (read-char in nil nil) + while char + do (write-string (who:escape-char-minimal char) out))) + +(defun cid-header-value (cid) + "Takes a Content-ID as present in Apple Notes' <object> tags and properly + surrounds them with angle brackets for a MIME header" + (concatenate 'string "<" cid ">")) + +;;; main implementation + +;; TODO(sterni): make this a “parser” instead of a predicate +(defun apple-note-p (msg) + "Checks X-Uniform-Type-Identifier of a MIME:MIME-MESSAGE + to determine if a given mime message is an Apple Note." + (when-let (uniform-id (assoc "X-Uniform-Type-Identifier" + (mime:mime-message-headers msg) + :test #'string=)) + (string= (cdr uniform-id) "com.apple.mail-note"))) + +(defun apple-note-html-fragment (msg out) + "Takes a MIME:MIME-MESSAGE and writes its text content as HTML to + the OUT stream. The <object> tags are resolved to <img> which + refer to the respective attachment's filename as a relative path, + but extraction of the attachments must be done separately. The + surrounding <html> and <body> tags are stripped and <head> + discarded completely, so only a fragment which can be included + in custom templates remains." + (let ((text (find-mime-text-part msg))) + (cond + ;; Sanity checking of the note + ((not (apple-note-p msg)) + (error "Unsupported or missing X-Uniform-Type-Identifier")) + ((not text) (error "Malformed Apple Note: no text part")) + ;; notemap creates text/plain notes we need to handle properly. + ;; Additionally we *could* check X-Mailer which notemap sets + ((string= (mime:mime-subtype text) "plain") + (html-escape-stream (mime:mime-body-stream text :binary nil) out)) + ;; Notes.app creates text/html parts + ((string= (mime:mime-subtype text) "html") + (closure-html:parse + (mime:mime-body-stream text) + (make-instance + 'apple-note-transformer + :cid-lookup + (lambda (cid) + (when-let* ((part (mime:find-mime-part-by-id msg (cid-header-value cid))) + (file (mime:mime-part-file-name part))) + file)) + :next-handler + (closure-html:make-character-stream-sink out)))) + (t (error "Malformed Apple Note: unknown mime type"))))) diff --git a/users/sterni/mblog/packages.lisp b/users/sterni/mblog/packages.lisp new file mode 100644 index 000000000000..ca2e41b6827f --- /dev/null +++ b/users/sterni/mblog/packages.lisp @@ -0,0 +1,15 @@ +(defpackage :mblog + (:use + :common-lisp + :mime4cl + :closure-html + :who + :uiop) + (:shadow :with-html-output) ; conflict between closure-html and who + (:import-from + :alexandria + :when-let* + :when-let + :starts-with-subseq + :ends-with-subseq) + (:export :main)) diff --git a/users/sterni/mblog/transformer.lisp b/users/sterni/mblog/transformer.lisp new file mode 100644 index 000000000000..f26c5652a266 --- /dev/null +++ b/users/sterni/mblog/transformer.lisp @@ -0,0 +1,127 @@ +(in-package :mblog) +(declaim (optimize (safety 3))) + +;; Throw away these tags and all of their children +(defparameter +discard-tags-with-children+ '("HEAD")) +;; Only “strip” these tags and leave their content as is +(defparameter +discard-tags-only+ '("BODY" "HTML")) + +;; This is basically the same as cxml's PROXY-HANDLER. +;; Couldn't be bothered to make a BROADCAST-HANDLER because I +;; only need to pass through to one handler. It accepts every +;; event and passes it on to NEXT-HANDLER. This is useful for +;; subclassing mostly where an event can be modified or passed +;; on as is via CALL-NEXT-METHOD. +(defclass hax-proxy-handler (hax:default-handler) + ((next-handler + :initarg :next-handler + :accessor proxy-next-handler))) + +;; Define the trivial handlers which just call themselves for NEXT-HANDLER +(macrolet ((def-proxy-handler (name (&rest args)) + `(defmethod ,name ((h hax-proxy-handler) ,@args) + (,name (proxy-next-handler h) ,@args)))) + (def-proxy-handler hax:start-document (name p-id s-id)) + (def-proxy-handler hax:end-document ()) + (def-proxy-handler hax:start-element (name attrs)) + (def-proxy-handler hax:end-element (name)) + (def-proxy-handler hax:characters (data)) + (def-proxy-handler hax:unescaped (data)) + (def-proxy-handler hax:comment (data))) + +(defclass apple-note-transformer (hax-proxy-handler) + ((cid-lookup + :initarg :cid-lookup + :initform (lambda (cid) nil) + :accessor transformer-cid-lookup) + (discard-until + :initarg :discard-until + :initform nil + :accessor transformer-discard-until) + (depth + :initarg :depth + :initform 0 + :accessor transformer-depth)) + (:documentation + "HAX handler that strips unnecessary tags from the HTML of a com.apple.mail-note + and resolves references to attachments to IMG tags.")) + +;; Define the “boring” handlers which just call the next method (i. e. the next +;; handler) unless discard-until is not nil in which case the event is dropped. +(macrolet ((def-filter-handler (name (&rest args)) + `(defmethod ,name ((h apple-note-transformer) ,@args) + (when (not (transformer-discard-until h)) + (call-next-method))))) + (def-filter-handler hax:start-document (name p-id s-id)) + (def-filter-handler hax:end-document ()) + (def-filter-handler hax:characters (data)) + (def-filter-handler hax:unescaped (data)) + (def-filter-handler hax:comment (data))) + +(defun parse-content-id (attrlist) + (when-let (data (find-if (lambda (x) + (string= (hax:attribute-name x) "DATA")) + attrlist)) + (multiple-value-bind (starts-with-cid-p suffix) + (starts-with-subseq "cid:" (hax:attribute-value data) + :return-suffix t :test #'char=) + (if starts-with-cid-p suffix data)))) + +(defmethod hax:start-element ((handler apple-note-transformer) name attrs) + (with-accessors ((discard-until transformer-discard-until) + (next-handler proxy-next-handler) + (cid-lookup transformer-cid-lookup) + (depth transformer-depth)) + handler + + (cond + ;; If we are discarding, any started element is dropped, + ;; since the end-condition only is reached via END-ELEMENT. + (discard-until nil) + ;; If we are not discarding any outer elements, we can set + ;; up a new discard condition if we encounter an appropriate + ;; element. + ((member name +discard-tags-with-children+ :test #'string=) + (setf discard-until (cons name depth))) + ;; Only drop this event, must be mirrored in END-ELEMENT to + ;; avoid invalidly nested HTML. + ((member name +discard-tags-only+ :test #'string=) nil) + ;; If we encounter an object tag, we drop it and its contents, + ;; but only after inspecting its attributes and emitting new + ;; events representing an img tag which includes the respective + ;; attachment via its filename. + ((string= name "OBJECT") + (progn + (setf discard-until (cons "OBJECT" depth)) + ;; TODO(sterni): check type and only resolve images, raise error + ;; otherwise. We should only encounter images anyways, since + ;; other types are only supported for iCloud which doesn't seem + ;; to use IMAP for sync these days. + (when-let* ((cid (parse-content-id attrs)) + (file (apply cid-lookup (list cid))) + (src (hax:make-attribute "SRC" file))) + (hax:start-element next-handler "IMG" (list src)) + (hax:end-element next-handler "IMG")))) + ;; In all other cases, we use HAX-PROXY-HANDLER to pass the event on. + (t (call-next-method))) + (setf depth (1+ depth)))) + +(defmethod hax:end-element ((handler apple-note-transformer) name) + (with-accessors ((discard-until transformer-discard-until) + (depth transformer-depth)) + handler + + (setf depth (1- depth)) + (cond + ;; If we are discarding and encounter the same tag again at the same + ;; depth, we can stop, but still have to discard the current tag. + ((and discard-until + (string= (car discard-until) name) + (= (cdr discard-until) depth)) + (setf discard-until nil)) + ;; In all other cases, we drop properly. + (discard-until nil) + ;; Mirrored tag stripping as in START-ELEMENT + ((member name +discard-tags-only+ :test #'string=) nil) + ;; In all other cases, we use HAX-PROXY-HANDLER to pass the event on. + (t (call-next-method))))) diff --git a/users/sterni/nix/char/all-chars.bin b/users/sterni/nix/char/all-chars.bin new file mode 100644 index 000000000000..017b909e8e8e --- /dev/null +++ b/users/sterni/nix/char/all-chars.bin @@ -0,0 +1,2 @@ + + !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ \ No newline at end of file diff --git a/users/sterni/nix/char/default.nix b/users/sterni/nix/char/default.nix new file mode 100644 index 000000000000..aacfc9dcbe4d --- /dev/null +++ b/users/sterni/nix/char/default.nix @@ -0,0 +1,95 @@ +{ depot, lib, pkgs, ... }: + +let + + inherit (depot.users.sterni.nix.flow) + cond + ; + + inherit (depot.nix) + yants + ; + + inherit (depot.users.sterni.nix) + string + ; + + # A char is the atomic element of a nix string + # which is essentially an array of arbitrary bytes + # as long as they are not a NUL byte. + # + # A char is neither a byte nor a unicode codepoint! + char = yants.restrict "char" (s: builtins.stringLength s == 1) yants.string; + + # integer representation of char + charval = yants.restrict "charval" (i: i >= 1 && i < 256) yants.int; + + allChars = builtins.readFile ./all-chars.bin; + + # Originally I searched a list for this, but came to the + # conclusion that this can never be fast enough in Nix. + # We therefore use a solution similar to infinisil's. + ordMap = builtins.listToAttrs + (lib.imap1 (i: v: { name = v; value = i; }) + (string.toChars allChars)); + + # Note on performance: + # chr and ord have been benchmarked using the following cases: + # + # builtins.map ord (lib.stringToCharacters allChars) + # builtins.map chr (builtins.genList (int.add 1) 255 + # + # The findings are as follows: + # 1. Searching through either strings using recursion is + # unbearably slow in Nix, leading to evaluation times + # of up to 3s for the following very small test case. + # This is why we use the trusty attribute set for ord. + # 2. String indexing is much faster than list indexing which + # is why we use the former for chr. + ord = c: ordMap."${c}"; + + chr = i: string.charAt (i - 1) allChars; + + asciiAlpha = c: + let + v = ord c; + in (v >= 65 && v <= 90) + || (v >= 97 && v <= 122); + + asciiNum = c: + let + v = ord c; + in v >= 48 && v <= 57; + + asciiAlphaNum = c: asciiAlpha c || asciiNum c; + +in { + inherit + allChars + char + charval + ord + chr + asciiAlpha + asciiNum + asciiAlphaNum + ; + + # originally I generated a nix file containing a list of + # characters, but infinisil uses a better way which I adapt + # which is using builtins.readFile instead of import. + __generateAllChars = pkgs.runCommandCC "generate-all-chars" { + source = '' + #include <stdio.h> + + int main(void) { + for(int i = 1; i <= 0xff; i++) { + putchar(i); + } + } + ''; + passAsFile = [ "source" ]; + } '' + $CC -o "$out" -x c "$sourcePath" + ''; +} diff --git a/users/sterni/nix/char/tests/default.nix b/users/sterni/nix/char/tests/default.nix new file mode 100644 index 000000000000..49b439adbb84 --- /dev/null +++ b/users/sterni/nix/char/tests/default.nix @@ -0,0 +1,31 @@ +{ depot, ... }: + +let + inherit (depot.nix.runTestsuite) + it + assertEq + runTestsuite + ; + + inherit (depot.users.sterni.nix) + char + string + int + fun + ; + + charList = string.toChars char.allChars; + + testAllCharConversion = it "tests conversion of all chars" [ + (assertEq "char.chr converts to char.allChars" + (builtins.genList (fun.rl char.chr (int.add 1)) 255) + charList) + (assertEq "char.ord converts from char.allChars" + (builtins.genList (int.add 1) 255) + (builtins.map char.ord charList)) + ]; + +in + runTestsuite "char" [ + testAllCharConversion + ] diff --git a/users/sterni/nix/flow/default.nix b/users/sterni/nix/flow/default.nix new file mode 100644 index 000000000000..b5783bd86deb --- /dev/null +++ b/users/sterni/nix/flow/default.nix @@ -0,0 +1,82 @@ +{ depot, ... }: + +let + + inherit (depot.nix) + yants + ; + + inherit (depot.users.sterni.nix) + fun + ; + + # we must avoid evaluating any of the sublists + # as they may contain conditions that throw + condition = yants.restrict "condition" + (ls: builtins.length ls == 2) + (yants.list yants.any); + + /* Like the common lisp macro: takes a list + of two elemented lists whose first element + is a boolean. The second element of the + first list that has true as its first + element is returned. + + Type: [ [ bool a ] ] -> a + + Example: + + cond [ + [ (builtins.isString true) 12 ] + [ (3 == 2) 13 ] + [ true 42 ] + ] + + => 42 + */ + cond = conds: switch true conds; + + /* Generic pattern match-ish construct for nix. + Takes a bunch of lists which are of length + two and checks the first element for either + a predicate or a value. The second value of + the first list which either has a value equal + to or a function that evaluates to true for + the given value. + + Type: a -> [ [ (function | a) b ] ] -> b + + Example: + + switch "foo" [ + [ "smol" "SMOL!!!" ] + [ (x: builtins.stringLength x <= 3) "smol-ish" ] + [ (fun.const true) "not smol" ] + ] + + => "smol-ish" + */ + switch = x: conds: + if builtins.length conds == 0 + then builtins.throw "exhausted all conditions" + else + let + c = condition (builtins.head conds); + s = builtins.head c; + b = + if builtins.isFunction s + then s x + else x == s; + in + if b + then builtins.elemAt c 1 + else switch x (builtins.tail conds); + + + +in { + inherit + cond + switch + ; +} diff --git a/users/sterni/nix/flow/tests/default.nix b/users/sterni/nix/flow/tests/default.nix new file mode 100644 index 000000000000..54cea01858e7 --- /dev/null +++ b/users/sterni/nix/flow/tests/default.nix @@ -0,0 +1,39 @@ +{ depot, ... }: + +let + + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + assertThrows + ; + + inherit (depot.users.sterni.nix.flow) + cond + match + ; + + dontEval = builtins.throw "this should not get evaluated"; + + testCond = it "tests cond" [ + (assertThrows "malformed cond list" + (cond [ [ true 1 2 ] [ false 1 ] ])) + (assertEq "last is true" "last" + (cond [ + [ false dontEval] + [ false dontEval ] + [ true "last" ] + ])) + (assertEq "first is true" 1 + (cond [ + [ true 1 ] + [ true dontEval ] + [ true dontEval ] + ])) + ]; + +in + runTestsuite "nix.flow" [ + testCond + ] diff --git a/users/sterni/nix/fun/default.nix b/users/sterni/nix/fun/default.nix new file mode 100644 index 000000000000..6b3541ed4c65 --- /dev/null +++ b/users/sterni/nix/fun/default.nix @@ -0,0 +1,59 @@ +{ depot, lib, ... }: + +let + + inherit (lib) + id + ; + + # Simple function composition, + # application is right to left. + rl = f1: f2: + (x: f1 (f2 x)); + + # Compose a list of functions, + # application is right to left. + rls = fs: + builtins.foldl' (fOut: f: lr f fOut) id fs; + + # Simple function composition, + # application is left to right. + lr = f1: f2: + (x: f2 (f1 x)); + + # Compose a list of functions, + # application is left to right + lrs = x: fs: + builtins.foldl' (v: f: f v) x fs; + + # Warning: cursed function + # + # Check if a function has an attribute + # set pattern with an ellipsis as its argument. + # + # s/o to puck for discovering that you could use + # builtins.toXML to introspect functions more than + # you should be able to in Nix. + hasEllipsis = f: + builtins.isFunction f && + builtins.match ".*<attrspat ellipsis=\"1\">.*" + (builtins.toXML f) != null; + +in + +{ + inherit (lib) + fix + flip + const + ; + + inherit + id + rl + rls + lr + lrs + hasEllipsis + ; +} diff --git a/users/sterni/nix/fun/tests/default.nix b/users/sterni/nix/fun/tests/default.nix new file mode 100644 index 000000000000..6492554306e1 --- /dev/null +++ b/users/sterni/nix/fun/tests/default.nix @@ -0,0 +1,29 @@ +{ depot, ... }: + +let + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + ; + + inherit (depot.users.sterni.nix) + fun + ; + + hasEllipsisTests = it "checks fun.hasEllipsis" [ + (assertEq "Malicious string" false + (fun.hasEllipsis (builtins.toXML ({ foo, ... }: 12)))) + (assertEq "No function" false + (fun.hasEllipsis 23)) + (assertEq "No attribute set pattern" false + (fun.hasEllipsis (a: a + 2))) + (assertEq "No ellipsis" false + (fun.hasEllipsis ({ foo, bar }: foo + bar))) + (assertEq "Ellipsis" true + (fun.hasEllipsis ({ depot, pkgs, ... }: 42))) + ]; +in + runTestsuite "nix.fun" [ + hasEllipsisTests + ] diff --git a/users/sterni/nix/html/README.md b/users/sterni/nix/html/README.md new file mode 100644 index 000000000000..0349e466a166 --- /dev/null +++ b/users/sterni/nix/html/README.md @@ -0,0 +1,148 @@ +# html.nix — _the_ most cursed Nix HTML DSL + +A quick example to show you what it looks like: + +```nix +# Note: this example is for standalone usage out of depot +{ pkgs ? import <nixpkgs> {} }: + +let + # zero dependency, one file implementation + htmlNix = import ./path/to/html.nix { }; + + # make the magic work + inherit (htmlNix) __findFile esc withDoctype; +in + +pkgs.writeText "example.html" (withDoctype (<html> {} [ + (<head> {} [ + (<meta> { charset = "utf-8"; } null) + (<title> {} (esc "hello world")) + ]) + (<body> {} [ + (<h1> {} (esc "hello world")) + (<p> { class = "intro"; } (esc '' + welcome to the land of sillyness! + '')) + (<ul> {} [ + (<li> {} [ + (esc "check out ") + (<a> { href = "https://code.tvl.fyi"; } "depot") + ]) + (<li> {} [ + (esc "find ") + (<a> { href = "https://cl.tvl.fyi/q/hashtag:cursed"; } "cursed things") + ]) + ]) + ]) +])) +``` + +Convince yourself it works: + +```console +$ $BROWSER $(nix-build example.nix) +``` + +Alternatively, in depot: + +```console +$ $BROWSER $(nix-build -A users.sterni.nix.html.tests) +``` + +## Creating tags + +An empty tag is passed `null` as its content argument: + +```nix +<link> { + rel = "stylesheet"; + href = "/main.css"; + type = "text/css"; +} null + +# => "<link href=\"/main.css\" rel=\"stylesheet\" type=\"text/css\"/>" +``` + +Content is expected to be HTML: + +```nix +<div> { class = "foo"; } "<strong>hi</strong>" + +# => "<div class=\"foo\"><strong>hi</strong></div>" +``` + +If it's not, be sure to escape it: + +```nix +<p> {} (esc "A => B") + +# => "<p>A => B</p>" +``` + +Nesting tags works of course: + +```nix +<div> {} (<strong> {} (<em> {} "hi")) + +# => "<div><strong><em>hi</em></strong></div>" +``` + +If the content of a tag is a list, it's concatenated: + +```nix +<h1> {} [ + (esc "The ") + (<strong> {} "Nix") + (esc " ") + (<em> {} "Expression") + (esc " Language") +] + +# => "<h1>The <strong>Nix</strong> <em>Expression</em> Language</h1>" +``` + +More detailed documentation can be found in `nixdoc`-compatible +comments in the source file (`default.nix` in this directory). + +## How does this work? + +*Theoretically* expressions like `<nixpkgs>` are just ordinary paths — +their actual value is determined from `NIX_PATH`. `html.nix` works +because of how this is actually implemented: At [parse time][spath-parsing] +Nix transparently translates an expression like `<foo>` into +`__findFile __nixPath "foo"`: + +``` +nix-repl> <nixpkgs> +/nix/var/nix/profiles/per-user/root/channels/vuizvui/nixpkgs + +nix-repl> __findFile __nixPath "nixpkgs" +/nix/var/nix/profiles/per-user/root/channels/vuizvui/nixpkgs +``` + +This translation doesn't take any scoping issues into account -- +so we can just shadow `__findFile` and make it return anything, +even a function: + +``` +nix-repl> __findFile = nixPath: str: + /**/ if str == "double" then x: x * 2 + else if str == "triple" then x: x * 3 + else throw "what?" + +nix-repl> <double> 2 +4 + +nix-repl> <triple> 3 +9 + +nix-repl> <quadruple> 4 +error: what? +``` + +Exactly this is what we are doing in `html.nix`: +Using `let inherit (htmlNix) __findFile; in` we shadow the builtin `__findFile` +with a function which returns a function rendering a particular HTML tag. + +[spath-parsing]: https://github.com/NixOS/nix/blob/293220bed5a75efc963e33c183787e87e55e28d9/src/libexpr/parser.y#L410-L416 diff --git a/users/sterni/nix/html/default.nix b/users/sterni/nix/html/default.nix new file mode 100644 index 000000000000..2498d832aadf --- /dev/null +++ b/users/sterni/nix/html/default.nix @@ -0,0 +1,119 @@ +# Copyright © 2021 sterni +# SPDX-License-Identifier: MIT +# +# This file provides a cursed HTML DSL for nix which works by overloading +# the NIX_PATH lookup operation via angle bracket operations, e. g. `<nixpkgs>`. + +{ ... }: + +let + /* Escape everything we have to escape in an HTML document if either + in a normal context or an attribute string (`<>&"'`). + + A shorthand for this function called `esc` is also provided. + + Type: string -> string + + Example: + + escapeMinimal "<hello>" + => "<hello>" + */ + escapeMinimal = builtins.replaceStrings + [ "<" ">" "&" "\"" "'" ] + [ "<" ">" "&" """ "'" ]; + + /* Return a string with a correctly rendered tag of the given name, + with the given attributes which are automatically escaped. + + If the content argument is `null`, the tag will have no children nor a + closing element. If the content argument is a string it is used as the + content as is (unescaped). If the content argument is a list, its + elements are concatenated. + + `renderTag` is only an internal function which is reexposed as `__findFile` + to allow for much neater syntax than calling `renderTag` everywhere: + + ```nix + { depot, ... }: + let + inherit (depot.users.sterni.nix.html) __findFile esc; + in + + <html> {} [ + (<head> {} (<title> {} (esc "hello world"))) + (<body> {} [ + (<h1> {} (esc "hello world")) + (<p> {} (esc "foo bar")) + ]) + ] + + ``` + + As you can see, the need to call a function disappears, instead the + `NIX_PATH` lookup operation via `<foo>` is overloaded, so it becomes + `renderTag "foo"` automatically. + + Since the content argument may contain the result of other `renderTag` + calls, we can't escape it automatically. Instead this must be done manually + using `esc`. + + Type: string -> attrs<string> -> (list<string> | string | null) -> string + + Example: + + <link> { + rel = "stylesheet"; + href = "/css/main.css"; + type = "text/css"; + } null + + renderTag "link" { + rel = "stylesheet"; + href = "/css/main.css"; + type = "text/css"; + } null + + => "<link href=\"/css/main.css\" rel=\"stylesheet\" type=\"text/css\"/>" + + <p> {} [ + "foo " + (<strong> {} "bar") + ] + + renderTag "p" {} "foo <strong>bar</strong>" + => "<p>foo <strong>bar</strong></p>" + */ + renderTag = tag: attrs: content: + let + attrs' = builtins.concatStringsSep "" ( + builtins.map (n: + " ${escapeMinimal n}=\"${escapeMinimal (toString attrs.${n})}\"" + ) (builtins.attrNames attrs) + ); + content' = + if builtins.isList content + then builtins.concatStringsSep "" content + else content; + in + if content == null + then "<${tag}${attrs'}/>" + else "<${tag}${attrs'}>${content'}</${tag}>"; + + /* Prepend "<!DOCTYPE html>" to a string. + + Type: string -> string + + Example: + + withDoctype (<body> {} (esc "hello")) + => "<!DOCTYPE html><body>hello</body>" + */ + withDoctype = doc: "<!DOCTYPE html>" + doc; + +in { + inherit escapeMinimal renderTag withDoctype; + + __findFile = _: renderTag; + esc = escapeMinimal; +} diff --git a/users/sterni/nix/html/tests/default.nix b/users/sterni/nix/html/tests/default.nix new file mode 100644 index 000000000000..8688b6937130 --- /dev/null +++ b/users/sterni/nix/html/tests/default.nix @@ -0,0 +1,84 @@ +{ depot, pkgs, ... }: + +let + inherit (depot.users.sterni.nix.html) + __findFile + esc + withDoctype + ; + + exampleDocument = withDoctype (<html> { lang = "en"; } [ + (<head> {} [ + (<meta> { charset = "utf-8"; } null) + (<title> {} "html.nix example document") + (<link> { + rel = "license"; + href = "https://code.tvl.fyi/about/LICENSE"; + type = "text/html"; + } null) + (<style> {} (esc '' + hgroup h2 { + font-weight: normal; + } + + dd { + margin: 0; + } + '')) + ]) + (<body> {} [ + (<main> {} [ + (<hgroup> {} [ + (<h1> {} (esc "html.nix")) + (<h2> {} [ + (<em> {} "the") + (esc " most cursed HTML DSL ever!") + ]) + ]) + (<dl> {} [ + (<dt> {} [ + (esc "Q: Wait, it's all ") + (<a> { + href = "https://cl.tvl.fyi/q/hashtag:cursed"; + } (esc "cursed")) + (esc " nix hacks?") + ]) + (<dd> {} (esc "A: Always has been. 🔫")) + (<dt> {} (esc "Q: Why does this work?")) + (<dd> {} [ + (esc "Because nix ") + (<a> { + href = "https://github.com/NixOS/nix/blob/293220bed5a75efc963e33c183787e87e55e28d9/src/libexpr/parser.y#L410-L416"; + } (esc "translates ")) + (<a> { + href = "https://github.com/NixOS/nix/blob/293220bed5a75efc963e33c183787e87e55e28d9/src/libexpr/lexer.l#L100"; + } (esc "SPATH tokens")) + (esc " like ") + (<code> {} (esc "<nixpkgs>")) + (esc " into calls to ") + (<code> {} (esc "__findFile")) + (esc " in the ") + (<em> {} (esc "current")) + (esc " scope.") + ]) + ]) + ]) + ]) + ]); +in + +pkgs.runCommandNoCC "html.nix.html" { + passAsFile = [ "exampleDocument" ]; + inherit exampleDocument; + nativeBuildInputs = [ pkgs.html5validator ]; +} '' + set -x + test "${esc "<> && \" \'"}" = "<> && " '" + + # slow as hell unfortunately + html5validator "$exampleDocumentPath" + + mv "$exampleDocumentPath" "$out" + + set +x +'' diff --git a/users/sterni/nix/int/default.nix b/users/sterni/nix/int/default.nix new file mode 100644 index 000000000000..b3157571272f --- /dev/null +++ b/users/sterni/nix/int/default.nix @@ -0,0 +1,124 @@ +{ depot, lib, ... }: + +let + + # TODO(sterni): implement nix.float and figure out which of these + # functions can be split out into a common nix.num + # library. + + inherit (depot.users.sterni.nix) + string + ; + + inherit (builtins) + bitOr + bitAnd + bitXor + mul + div + add + sub + ; + + abs = i: if i < 0 then -i else i; + + exp = base: pow: + if pow > 0 + then base * (exp base (pow - 1)) + else if pow < 0 + then 1.0 / exp base (abs pow) + else 1; + + bitShiftR = bit: count: + if count == 0 + then bit + else div (bitShiftR bit (count - 1)) 2; + + bitShiftL = bit: count: + if count == 0 + then bit + else 2 * (bitShiftL bit (count - 1)); + + hexdigits = "0123456789ABCDEF"; + + toHex = int: + let + go = i: + if i == 0 + then "" + else go (bitShiftR i 4) + + string.charAt (bitAnd i 15) hexdigits; + sign = lib.optionalString (int < 0) "-"; + in + if int == 0 + then "0" + else "${sign}${go (abs int)}"; + + fromHexMap = builtins.listToAttrs + (lib.imap0 (i: c: { name = c; value = i; }) + (lib.stringToCharacters hexdigits)); + + fromHex = literal: + let + negative = string.charAt 0 literal == "-"; + start = if negative then 1 else 0; + len = builtins.stringLength literal; + # reversed list of all digits + digits = builtins.genList + (i: string.charAt (len - 1 - i) literal) + (len - start); + parsed = builtins.foldl' + (v: d: { + val = v.val + (fromHexMap."${d}" * v.mul); + mul = v.mul * 16; + }) + { val = 0; mul = 1; } digits; + in + if negative + then -parsed.val + else parsed.val; + + # A nix integer is a 64bit signed integer + maxBound = 9223372036854775807; + + # fun fact: -9223372036854775808 is the lower bound + # for a nix integer (as you would expect), but you can't + # use it as an integer literal or you'll be greeted with: + # error: invalid integer '9223372036854775808' + # This is because all int literals when parsing are + # positive, negative "literals" are positive literals + # which are preceded by the arithmetric negation operator. + minBound = -9223372036854775807 - 1; + + odd = x: bitAnd x 1 == 1; + even = x: bitAnd x 1 == 0; + + # div and mod behave like quot and rem in Haskell, + # i. e. they truncate towards 0 + mod = a: b: let res = a / b; in a - (res * b); + + inRange = a: b: x: x >= a && x <= b; + +in { + inherit + maxBound + minBound + abs + exp + odd + even + add + sub + mul + div + mod + bitShiftR + bitShiftL + bitOr + bitAnd + bitXor + toHex + fromHex + inRange + ; +} diff --git a/users/sterni/nix/int/tests/default.nix b/users/sterni/nix/int/tests/default.nix new file mode 100644 index 000000000000..fac45dd251e1 --- /dev/null +++ b/users/sterni/nix/int/tests/default.nix @@ -0,0 +1,203 @@ +{ depot, lib, ... }: + +let + + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + ; + + inherit (depot.users.sterni.nix) + int + string + fun + ; + + testBounds = it "checks minBound and maxBound" [ + # this is gonna blow up in my face because + # integer overflow is undefined behavior in + # C++, so most likely anything could happen? + (assertEq "maxBound is the maxBound" true + (int.maxBound + 1 < int.maxBound)) + (assertEq "minBound is the minBound" true + (int.minBound - 1 > int.minBound)) + (assertEq "maxBound overflows to minBound" + (int.maxBound + 1) + int.minBound) + (assertEq "minBound overflows to maxBound" + (int.minBound - 1) + int.maxBound) + ]; + + expectedBytes = [ + "00" "01" "02" "03" "04" "05" "06" "07" "08" "09" "0A" "0B" "0C" "0D" "0E" "0F" + "10" "11" "12" "13" "14" "15" "16" "17" "18" "19" "1A" "1B" "1C" "1D" "1E" "1F" + "20" "21" "22" "23" "24" "25" "26" "27" "28" "29" "2A" "2B" "2C" "2D" "2E" "2F" + "30" "31" "32" "33" "34" "35" "36" "37" "38" "39" "3A" "3B" "3C" "3D" "3E" "3F" + "40" "41" "42" "43" "44" "45" "46" "47" "48" "49" "4A" "4B" "4C" "4D" "4E" "4F" + "50" "51" "52" "53" "54" "55" "56" "57" "58" "59" "5A" "5B" "5C" "5D" "5E" "5F" + "60" "61" "62" "63" "64" "65" "66" "67" "68" "69" "6A" "6B" "6C" "6D" "6E" "6F" + "70" "71" "72" "73" "74" "75" "76" "77" "78" "79" "7A" "7B" "7C" "7D" "7E" "7F" + "80" "81" "82" "83" "84" "85" "86" "87" "88" "89" "8A" "8B" "8C" "8D" "8E" "8F" + "90" "91" "92" "93" "94" "95" "96" "97" "98" "99" "9A" "9B" "9C" "9D" "9E" "9F" + "A0" "A1" "A2" "A3" "A4" "A5" "A6" "A7" "A8" "A9" "AA" "AB" "AC" "AD" "AE" "AF" + "B0" "B1" "B2" "B3" "B4" "B5" "B6" "B7" "B8" "B9" "BA" "BB" "BC" "BD" "BE" "BF" + "C0" "C1" "C2" "C3" "C4" "C5" "C6" "C7" "C8" "C9" "CA" "CB" "CC" "CD" "CE" "CF" + "D0" "D1" "D2" "D3" "D4" "D5" "D6" "D7" "D8" "D9" "DA" "DB" "DC" "DD" "DE" "DF" + "E0" "E1" "E2" "E3" "E4" "E5" "E6" "E7" "E8" "E9" "EA" "EB" "EC" "ED" "EE" "EF" + "F0" "F1" "F2" "F3" "F4" "F5" "F6" "F7" "F8" "F9" "FA" "FB" "FC" "FD" "FE" "FF" + ]; + + hexByte = i: string.fit { width = 2; char = "0"; } (int.toHex i); + + hexInts = [ + { left = 0; right = "0"; } + { left = 1; right = "1"; } + { left = 11; right = "B"; } + { left = 123; right = "7B"; } + { left = 9000; right = "2328"; } + { left = 2323; right = "913"; } + { left = 4096; right = "1000"; } + { left = int.maxBound; right = "7FFFFFFFFFFFFFFF"; } + { left = int.minBound; right = "-8000000000000000"; } + ]; + + testHex = it "checks conversion to hex" (lib.flatten [ + (lib.imap0 (i: hex: [ + (assertEq "hexByte ${toString i} == ${hex}" (hexByte i) hex) + (assertEq "${toString i} == fromHex ${hex}" i (int.fromHex hex)) + ]) expectedBytes) + (builtins.map ({ left, right }: [ + (assertEq "toHex ${toString left} == ${right}" (int.toHex left) right) + (assertEq "${toString left} == fromHex ${right}" left (int.fromHex right)) + ]) hexInts) + ]); + + testBasic = it "checks basic int operations" [ + (assertEq "122 is even" (int.even 122 && !(int.odd 122)) true) + (assertEq "123 is odd" (int.odd 123 && !(int.even 123)) true) + (assertEq "abs -4959" (int.abs (-4959)) 4959) + ]; + + expNumbers = [ + { left = -3; right = 0.125; } + { left = -2; right = 0.25; } + { left = -1; right = 0.5; } + { left = 0; right = 1; } + { left = 1; right = 2; } + { left = 2; right = 4; } + { left = 3; right = 8; } + { left = 4; right = 16; } + { left = 5; right = 32; } + { left = 16; right = 65536; } + ]; + + testExp = it "checks exponentiation" + (builtins.map ({ left, right }: + assertEq + "2 ^ ${toString left} == ${toString right}" + (int.exp 2 left) right) expNumbers); + + shifts = [ + { a = 2; b = 5; c = 64; op = "<<"; } + { a = -2; b = 5; c = -64; op = "<<"; } + { a = 123; b = 4; c = 1968; op = "<<"; } + { a = 1; b = 8; c = 256; op = "<<"; } + { a = 256; b = 8; c = 1; op = ">>"; } + { a = 374; b = 2; c = 93; op = ">>"; } + { a = 2; b = 2; c = 0; op = ">>"; } + { a = 99; b = 9; c = 0; op = ">>"; } + ]; + + checkShift = { a, b, c, op }@args: + let + f = string.match op { + "<<" = int.bitShiftL; + ">>" = int.bitShiftR; + }; + in assertEq "${toString a} ${op} ${toString b} == ${toString c}" (f a b) c; + + checkShiftRDivExp = n: + assertEq "${toString n} >> 5 == ${toString n} / 2 ^ 5" + (int.bitShiftR n 5) (int.div n (int.exp 2 5)); + + checkShiftLMulExp = n: + assertEq "${toString n} >> 6 == ${toString n} * 2 ^ 6" + (int.bitShiftL n 5) (int.mul n (int.exp 2 5)); + + testBit = it "checks bitwise operations" (lib.flatten [ + (builtins.map checkShift shifts) + (builtins.map checkShiftRDivExp [ + 1 + 2 + 3 + 5 + 7 + 23 + 1623 + 238 + 34 + 348 + 2834 + 834 + 348 + ]) + (builtins.map checkShiftLMulExp [ + 1 + 2 + 3 + 5 + 7 + 23 + 384 + 3 + 2 + 5991 + 85109 + 38 + ]) + ]); + + divisions = [ + { a = 2; b = 1; c = 2; mod = 0;} + { a = 2; b = 2; c = 1; mod = 0;} + { a = 20; b = 10; c = 2; mod = 0;} + { a = 12; b = 5; c = 2; mod = 2;} + { a = 23; b = 4; c = 5; mod = 3;} + ]; + + checkDiv = n: { a, b, c, mod }: [ + (assertEq "${n}: div result" (int.div a b) c) + (assertEq "${n}: mod result" (int.mod a b) mod) + (assertEq "${n}: divMod law" ((int.div a b) * b + (int.mod a b)) a) + ]; + + testDivMod = it "checks integer division and modulo" + (lib.flatten [ + (builtins.map (checkDiv "+a / +b") divisions) + (builtins.map (fun.rl (checkDiv "-a / +b") (x: x // { + a = -x.a; + c = -x.c; + mod = -x.mod; + })) divisions) + (builtins.map (fun.rl (checkDiv "+a / -b") (x: x // { + b = -x.b; + c = -x.c; + })) divisions) + (builtins.map (fun.rl (checkDiv "-a / -b") (x: x // { + a = -x.a; + b = -x.b; + mod = -x.mod; + })) divisions) + ]); + +in + runTestsuite "nix.int" [ + testBounds + testHex + testBasic + testExp + testBit + testDivMod + ] diff --git a/users/sterni/nix/string/default.nix b/users/sterni/nix/string/default.nix new file mode 100644 index 000000000000..19d2cec243c0 --- /dev/null +++ b/users/sterni/nix/string/default.nix @@ -0,0 +1,114 @@ +{ depot, lib, ... }: + +let + + inherit (depot.users.sterni.nix.char) + chr + ord + ; + + inherit (depot.users.sterni.nix) + int + flow + ; + + take = n: s: + builtins.substring 0 n s; + + drop = n: s: + builtins.substring n int.maxBound s; + + charAt = i: s: + let + r = builtins.substring i 1 s; + in if r == "" then null else r; + + charIndex = char: s: + let + len = builtins.stringLength s; + go = i: + flow.cond [ + [ (i >= len) null ] + [ (charAt i s == char) i ] + [ true (go (i + 1)) ] + ]; + in go 0; + + toChars = lib.stringToCharacters; + fromChars = lib.concatStrings; + + toBytes = str: + builtins.map ord (toChars str); + + fromBytes = is: lib.concatMapStrings chr is; + + pad = { left ? 0, right ? 0, char ? " " }: s: + let + leftS = fromChars (builtins.genList (_: char) left); + rightS = fromChars (builtins.genList (_: char) right); + in "${leftS}${s}${rightS}"; + + fit = { char ? " ", width, side ? "left" }: s: + let + diff = width - builtins.stringLength s; + in + if diff <= 0 + then s + else pad { inherit char; "${side}" = diff; } s; + + # pattern matching for strings only + match = val: matcher: matcher."${val}"; + + /* Bare-bones printf implementation. Supported format specifiers: + + * `%%` escapes `%` + * `%s` is substituted by a string + + As expected, the first argument is a format string and the values + for its format specifiers need to provided as the next arguments + in order. + + Type: string -> (printfVal : either string (a -> printfVal)) + */ + printf = formatString: + let + specifierWithArg = token: builtins.elem token [ + "%s" + ]; + isSpecifier = lib.hasPrefix "%"; + + tokens = lib.flatten (builtins.split "(%.)" formatString); + argsNeeded = builtins.length (builtins.filter specifierWithArg tokens); + + format = args: (builtins.foldl' ({ out ? "", argIndex ? 0 }: token: { + argIndex = argIndex + (if specifierWithArg token then 1 else 0); + out = + /**/ if token == "%s" then out + builtins.elemAt args argIndex + else if token == "%%" then out + "%" + else if isSpecifier token then throw "Unsupported format specifier ${token}" + else out + token; + }) {} tokens).out; + + accumulateArgs = argCount: args: + if argCount > 0 + then arg: accumulateArgs (argCount - 1) (args ++ [ arg ]) + else format args; + in + accumulateArgs argsNeeded []; + +in { + inherit + take + drop + charAt + charIndex + toBytes + fromBytes + toChars + fromChars + pad + fit + match + printf + ; +} diff --git a/users/sterni/nix/string/tests/default.nix b/users/sterni/nix/string/tests/default.nix new file mode 100644 index 000000000000..c8aec9464077 --- /dev/null +++ b/users/sterni/nix/string/tests/default.nix @@ -0,0 +1,72 @@ +{ depot, ... }: + +let + + inherit (depot.users.sterni.nix) + string + ; + + inherit (depot.nix.runTestsuite) + it + assertEq + runTestsuite + ; + + testTakeDrop = it "tests take and drop" [ + (assertEq "take" + (string.take 5 "five and more") + "five ") + (assertEq "drop" + (string.drop 2 "coin") + "in") + (assertEq "take out of bounds" + (string.take 100 "foo") + "foo") + (assertEq "drop out of bounds" + (string.drop 42 "lol") + "") + ]; + + testIndexing = it "tests string indexing" [ + (assertEq "normal charAt" + (string.charAt 3 "helo") + "o") + (assertEq "out of bounds charAt" + (string.charAt 5 "helo") + null) + ]; + + testFinding = it "tests finding in strings" [ + (assertEq "normal charIndex" + (string.charIndex "d" "abcdefghijkl") + 3) + (assertEq "charIndex no match" + (string.charIndex "w" "zZzZzzzZZZ") + null) + ]; + + dontEval = builtins.throw "this should not get evaluated"; + + testMatch = it "tests match" [ + (assertEq "basic match usage" 42 + (string.match "answer" { + "answer" = 42; + "banana" = dontEval; + "maleur" = dontEval; + })) + ]; + + f = "f"; + testPrintf = it "prints f" [ + (assertEq "basic %s usage" "print ${f}" (string.printf "print %s" f)) + (assertEq "% escaping" "100%" (string.printf "100%%")) + ]; + +in + runTestsuite "nix.string" [ + testTakeDrop + testIndexing + testFinding + testMatch + testPrintf + ] diff --git a/users/sterni/nix/url/default.nix b/users/sterni/nix/url/default.nix new file mode 100644 index 000000000000..37bd0de66ac9 --- /dev/null +++ b/users/sterni/nix/url/default.nix @@ -0,0 +1,81 @@ +{ depot, lib, ... }: + +let + + inherit (depot.users.sterni.nix) + char + int + string + flow + ; + + reserved = c: builtins.elem c [ + "!" "#" "$" "&" "'" "(" ")" + "*" "+" "," "/" ":" ";" "=" + "?" "@" "[" "]" + ]; + + unreserved = c: char.asciiAlphaNum c + || builtins.elem c [ "-" "_" "." "~" ]; + + percentEncode = c: + if unreserved c + then c + else "%" + (string.fit { + width = 2; + char = "0"; + side = "left"; + } (int.toHex (char.ord c))); + + encode = { leaveReserved ? false }: s: + let + chars = lib.stringToCharacters s; + tr = c: + if leaveReserved && reserved c + then c + else percentEncode c; + in lib.concatStrings (builtins.map tr chars); + + decode = s: + let + tokens = builtins.split "%" s; + decodeStep = + { result ? "" + , inPercent ? false + }: s: + flow.cond [ + [ + (builtins.isList s) + { + inherit result; + inPercent = true; + } + ] + [ + inPercent + { + inPercent = false; + # first two characters came after an % + # the rest is the string until the next % + result = result + + char.chr (int.fromHex (string.take 2 s)) + + (string.drop 2 s); + } + ] + [ + (!inPercent) + { + result = result + s; + } + ] + ]; + + in + (builtins.foldl' decodeStep {} tokens).result; + +in { + inherit + encode + decode + ; +} diff --git a/users/sterni/nix/url/tests/default.nix b/users/sterni/nix/url/tests/default.nix new file mode 100644 index 000000000000..7cf53cde1555 --- /dev/null +++ b/users/sterni/nix/url/tests/default.nix @@ -0,0 +1,56 @@ +{ depot, ... }: + +let + + inherit (depot.nix.runTestsuite) + it + assertEq + runTestsuite + ; + + inherit (depot.users.sterni.nix) + url + ; + + checkEncoding = args: { left, right }: + assertEq "encode ${builtins.toJSON left} == ${builtins.toJSON right}" + (url.encode args left) right; + + checkDecoding = { left, right }: + assertEq "${builtins.toJSON left} == decode ${builtins.toJSON right}" + (url.decode left) right; + + unreserved = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_~"; + + encodeExpected = [ + { left = "Laguna Beach"; right = "Laguna%20Beach"; } + { left = "👾 Exterminate!"; right = "%F0%9F%91%BE%20Exterminate%21"; } + { left = unreserved; right = unreserved; } + { + left = "`!@#$%^&*()+={}[]:;'\\|<>,?/ \""; + right = "%60%21%40%23%24%25%5E%26%2A%28%29%2B%3D%7B%7D%5B%5D%3A%3B%27%5C%7C%3C%3E%2C%3F%2F%20%22"; + } + ]; + + testEncode = it "checks url.encode" + (builtins.map (checkEncoding {}) encodeExpected); + + testDecode = it "checks url.decode" + (builtins.map checkDecoding encodeExpected); + + testLeaveReserved = it "checks that leaveReserved is like id for valid URLs" + (builtins.map (x: checkEncoding { leaveReserved = true; } { left = x; right = x; }) [ + "ftp://ftp.is.co.za/rfc/rfc1808.txt" + "http://www.ietf.org/rfc/rfc2396.txt" + "ldap://[2001:db8::7]/c=GB?objectClass?one" + "mailto:John.Doe@example.com" + "news:comp.infosystems.www.servers.unix" + "tel:+1-816-555-1212" + "telnet://192.0.2.16:80/" + "urn:oasis:names:specification:docbook:dtd:xml:4.1.2" + ]); +in + runTestsuite "nix.url" [ + testEncode + testLeaveReserved + ] diff --git a/users/sterni/nix/utf8/default.nix b/users/sterni/nix/utf8/default.nix new file mode 100644 index 000000000000..270da934b6a6 --- /dev/null +++ b/users/sterni/nix/utf8/default.nix @@ -0,0 +1,313 @@ +{ depot, lib, ... }: + +let + + inherit (depot.users.sterni.nix) + char + flow + fun + int + string + util + ; + + /* (Internal) function to determine the amount + bytes left in a UTF-8 byte sequence from the + first byte. + + This function will throw if the given first + byte is ill-formed, but will not detect all + cases of ill-formed-ness. + + Based on table 3-6. from The Unicode Standard, + Version 13.0, section 3.9. + + Type: integer -> integer + */ + byteCount = i: flow.cond [ + [ (int.bitAnd i 128 == 0) 1 ] + [ (int.bitAnd i 224 == 192) 2 ] + [ (int.bitAnd i 240 == 224) 3 ] + [ (int.bitAnd i 248 == 240) 4 ] + [ true (builtins.throw "Ill-formed first byte ${int.toHex i}") ] + ]; + + /* (Internal) function to check if a given byte in + an UTF-8 byte sequence is well-formed. + + Based on table 3-7. from The Unicode Standard, + Version 13.0, section 3.9. + + Type: integer -> integer -> integer -> bool + */ + wellFormedByte = + # first byte's integer value + first: + # byte position as an index starting with 0 + pos: + let + defaultRange = int.inRange 128 191; + + secondBytePredicate = flow.switch first [ + [ (int.inRange 194 223) defaultRange ] # C2..DF + [ 224 (int.inRange 160 191) ] # E0 + [ (int.inRange 225 236) defaultRange ] # E1..EC + [ 237 (int.inRange 128 159) ] # ED + [ (int.inRange 238 239) defaultRange ] # EE..EF + [ 240 (int.inRange 144 191) ] # F0 + [ (int.inRange 241 243) defaultRange ] # F1..F3 + [ 244 (int.inRange 128 143) ] # F4 + [ (fun.const true) null ] + ]; + + firstBytePredicate = byte: assert first == byte; + first < 128 || secondBytePredicate != null; + in + # Either ASCII or in one of the byte ranges of Table 3-6. + if pos == 0 then firstBytePredicate + # return predicate according to Table 3-6. + else if pos == 1 then assert secondBytePredicate != null; secondBytePredicate + # 3rd and 4th byte have only one validity rule + else defaultRange; + + /* Iteration step for decoding an UTF-8 byte sequence. + It decodes incrementally, i. e. it has to be fed + one byte at a time and then returns either a + new state or a final result. + + If the resulting attribute set contains the attribute + result, it is finished and the decoded codepoint is + contained in that attribute. In all other cases, + pass the returned set to step again along with + a new byte. The initial state to pass is the empty + set. + + Extra attributes are always passed through, so you + can pass extra state. Be sure not to use result, + pos, code, first or count. + + This function will throw with a fairly detailed + message if it encounters ill-formed bytes. + + The implementation is based on The Unicode Standard, + Version 13.0, section 3.9, especially table 3-6. + + Type: { ... } -> string -> ({ result :: integer, ... } | { ... }) + + Example: utf8.step {} "f" + => { result = 102; } + */ + step = { pos ? 0, code ? 0, ... }@args: byte: + let + value = char.ord byte; + # first byte is context for well-formed-ness + first = args.first or value; + count = args.count or (byteCount first); + newCode = + if count == 1 + then int.bitAnd 127 first # ascii character + else # multi byte UTF-8 sequence + let + # Calculate the bitmask for extracting the + # codepoint data in the current byte. + # If the codepoint is not ASCII, the bits + # used for codepoint data differ depending + # on the byte position and overall byte + # count. The first byte always ignores + # the (count + 1) most significant bits. + # For all subsequent bytes, the 2 most + # significant bits need to be ignored. + # See also table 3-6. + mask = + if pos == 0 + then int.exp 2 (8 - (count + 1)) - 1 + else 63; + # UTF-8 uses the 6 least significant bits in all + # subsequent bytes after the first one. Therefore + # We can determine the amount we need to shift + # the current value by the amount of bytes left. + offset = (count - (pos + 1)) * 6; + in + code + (int.bitShiftL (int.bitAnd mask value) offset); + illFormedMsg = + "Ill-formed byte ${int.toHex value} at position ${toString pos} in ${toString count} byte UTF-8 sequence"; + in + if !(wellFormedByte first pos value) then builtins.throw illFormedMsg + else if pos + 1 == count + then (builtins.removeAttrs args [ # allow extra state being passed through + "count" + "code" + "pos" + "first" + ]) // { result = newCode; } + else (builtins.removeAttrs args [ "result" ]) // { + inherit count first; + code = newCode; + pos = pos + 1; + }; + + /* Decode an UTF-8 string into a list of codepoints. + + Throws if the string is ill-formed UTF-8. + + Type: string -> [ integer ] + */ + # TODO(sterni): option to fallback to replacement char instead of failure + decode = s: + let + stringLength = builtins.stringLength s; + iterResult = builtins.genericClosure { + startSet = [ + { + key = "start"; + stringIndex = -1; + state = {}; + codepoint = null; + } + ]; + operator = { state, stringIndex, ... }: + let + # updated values for current iteration step + newIndex = stringIndex + 1; + newState = step state (builtins.substring newIndex 1 s); + in lib.optional (newIndex < stringLength) { + # unique keys to make genericClosure happy + key = toString newIndex; + # carryover state for the next step + stringIndex = newIndex; + state = newState; + # actual payload for later, steps with value null are filtered out + codepoint = newState.result or null; + }; + }; + in + # extract all steps that yield a code point into a list + builtins.map (v: v.codepoint) ( + builtins.filter ( + { codepoint, stringIndex, state, ... }: + + let + # error message in case we are missing bytes at the end of input + earlyEndMsg = + if state ? count && state ? pos + then "Missing ${toString (with state; count - pos)} bytes at end of input" + else "Unexpected end of input"; + in + + # filter out all iteration steps without a codepoint value + codepoint != null + # if we are at the iteration step of a non-empty input string, throw + # an error if no codepoint was returned, as it indicates an incomplete + # UTF-8 sequence. + || (stringLength > 0 && stringIndex == stringLength - 1 && throw earlyEndMsg) + + ) iterResult + ); + + /* Pretty prints a Unicode codepoint in the U+<HEX> notation. + + Type: integer -> string + */ + formatCodepoint = cp: "U+" + string.fit { + width = 4; + char = "0"; + } (int.toHex cp); + + encodeCodepoint = cp: + let + # Find the amount of bytes needed to encode the given codepoint. + # Note that this doesn't check if the Unicode codepoint is allowed, + # but rather allows all theoretically UTF-8-encodeable ones. + count = flow.switch cp [ + [ (int.inRange 0 127) 1 ] # 00000000 0xxxxxxx + [ (int.inRange 128 2047) 2 ] # 00000yyy yyxxxxxx + [ (int.inRange 2048 65535) 3 ] # zzzzyyyy yyxxxxxx + [ (int.inRange 65536 1114111) 4 ] # 000uuuuu zzzzyyyy yyxxxxxx, + # capped at U+10FFFF + + [ (fun.const true) (builtins.throw invalidCodepointMsg) ] + ]; + + invalidCodepointMsg = "${formatCodepoint cp} is not a Unicode codepoint"; + + # Extract the bit ranges x, y, z and u from the given codepoint + # according to Table 3-6. from The Unicode Standard, Version 13.0, + # section 3.9. u is split into uh and ul since they are used in + # different bytes in the end. + components = lib.mapAttrs (_: { mask, offset }: + int.bitAnd (int.bitShiftR cp offset) mask + ) { + x = { + mask = if count > 1 then 63 else 127; + offset = 0; + }; + y = { + mask = if count > 2 then 63 else 31; + offset = 6; + }; + z = { + mask = 15; + offset = 12; + }; + # u which belongs into the second byte + ul = { + mask = 3; + offset = 16; + }; + # u which belongs into the first byte + uh = { + mask = 7; + offset = 18; + }; + }; + inherit (components) x y z ul uh; + + # Finally construct the byte sequence for the given codepoint. This is + # usually done by using the component and adding a few bits as a prefix + # which depends on the length of the sequence. The longer the sequence, + # the further back each component is pushed. To simplify this, we + # always construct a 4 element list and take the last `count` elements. + # Thanks to laziness the bogus values created by this are never evaluated. + # + # Based on table 3-6. from The Unicode Standard, + # Version 13.0, section 3.9. + bytes = lib.sublist (4 - count) count [ + # 11110uuu + (uh + 240) + # 10uuzzzz or 1110zzzz + (z + (if count > 3 then 128 + int.bitShiftL ul 4 else 224)) + # 10yyyyyy or 110yyyyy + (y + (if count > 2 then 128 else 192)) + # 10xxxxxx or 0xxxxxxx + (x + (if count > 1 then 128 else 0)) + ]; + + firstByte = builtins.head bytes; + + unableToEncodeMessage = "Can't encode ${formatCodepoint cp} as UTF-8"; + + in string.fromBytes ( + builtins.genList (i: + let + byte = builtins.elemAt bytes i; + in + if wellFormedByte firstByte i byte + then byte + else builtins.throw unableToEncodeMessage + ) count + ); + + /* Encode a list of Unicode codepoints into an UTF-8 string. + + Type: [ integer ] -> string + */ + encode = lib.concatMapStrings encodeCodepoint; + +in { + inherit + encode + decode + step + formatCodepoint + ; +} diff --git a/users/sterni/nix/utf8/tests/default.nix b/users/sterni/nix/utf8/tests/default.nix new file mode 100644 index 000000000000..ddcd34208a6d --- /dev/null +++ b/users/sterni/nix/utf8/tests/default.nix @@ -0,0 +1,141 @@ +{ depot, pkgs, lib, ... }: + +let + + inherit (pkgs) + runCommandLocal + ; + + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + assertThrows + assertDoesNotThrow + ; + + inherit (depot.nix.writers) + rustSimple + ; + + inherit (depot.users.sterni.nix) + int + utf8 + string + char + ; + + rustDecoder = rustSimple { + name = "utf8-decode"; + } '' + use std::io::{self, Read}; + fn main() -> std::io::Result<()> { + let mut buffer = String::new(); + io::stdin().read_to_string(&mut buffer)?; + + print!("[ "); + + for c in buffer.chars() { + print!("{} ", u32::from(c)); + } + + print!("]"); + + Ok(()) + } + ''; + + rustDecode = s: + let + expr = runCommandLocal "${s}-decoded" {} '' + printf '%s' ${lib.escapeShellArg s} | ${rustDecoder} > $out + ''; + in import expr; + + hexDecode = l: + utf8.decode (string.fromBytes (builtins.map int.fromHex l)); + + hexEncode = l: utf8.encode (builtins.map int.fromHex l); + + testFailures = it "checks UTF-8 decoding failures" ([ + (assertThrows "truncated UTF-8 string throws" (hexDecode [ "F0" "9F" ])) + # examples from The Unicode Standard + (assertThrows "ill-formed: C0 AF" (hexDecode [ "C0" "AF" ])) + (assertThrows "ill-formed: E0 9F 80" (hexDecode [ "E0" "9F" "80" ])) + (assertEq "well-formed: F4 80 83 92" (hexDecode [ "F4" "80" "83" "92" ]) [ 1048786 ]) + (assertThrows "Codepoint out of range: 0xFFFFFF" (hexEncode [ "FFFFFF" ])) + (assertThrows "Codepoint out of range: -0x02" (hexEncode [ "-02" ])) + ] ++ builtins.genList (i: + let + cp = i + int.fromHex "D800"; + in + assertThrows "Can't encode UTF-16 reserved characters: ${utf8.formatCodepoint cp}" + (utf8.encode [ cp ]) + ) (int.fromHex "07FF")); + + testAscii = it "checks decoding of ascii strings" + (builtins.map (s: assertEq "ASCII decoding is equal to UTF-8 decoding for \"${s}\"" + (string.toBytes s) (utf8.decode s)) [ + "foo bar" + "hello\nworld" + "carriage\r\nreturn" + "1238398494829304 []<><>({})[]!!)" + (string.take 127 char.allChars) + ]); + + randomUnicode = [ + "" # empty string should yield empty list + "🥰👨👨👧👦🐈⬛👩🏽🦰" + # https://kermitproject.org/utf8.html + "ᚠᛇᚻ᛫ᛒᛦᚦ᛫ᚠᚱᚩᚠᚢᚱ᛫ᚠᛁᚱᚪ᛫ᚷᛖᚻᚹᛦᛚᚳᚢᛗ" + "An preost wes on leoden, Laȝamon was ihoten" + "Sîne klâwen durh die wolken sint geslagen," + "Τὴ γλῶσσα μοῦ ἔδωσαν ἑλληνικὴ" + "На берегу пустынных волн" + "ვეპხის ტყაოსანი შოთა რუსთაველი" + "யாமறிந்த மொழிகளிலே தமிழ்மொழி போல் இனிதாவது எங்கும் காணோம், " + "ಬಾ ಇಲ್ಲಿ ಸಂಭವಿಸು " + ]; + + # https://kermitproject.org/utf8.html + glassSentences = [ + "Euro Symbol: €." + "Greek: Μπορώ να φάω σπασμένα γυαλιά χωρίς να πάθω τίποτα." + "Íslenska / Icelandic: Ég get etið gler án þess að meiða mig." + "Polish: Mogę jeść szkło, i mi nie szkodzi." + "Romanian: Pot să mănânc sticlă și ea nu mă rănește." + "Ukrainian: Я можу їсти шкло, й воно мені не пошкодить." + "Armenian: Կրնամ ապակի ուտել և ինծի անհանգիստ չըներ։" + "Georgian: მინას ვჭამ და არა მტკივა." + "Hindi: मैं काँच खा सकता हूँ, मुझे उस से कोई पीडा नहीं होती." + "Hebrew(2): אני יכול לאכול זכוכית וזה לא מזיק לי." + "Yiddish(2): איך קען עסן גלאָז און עס טוט מיר נישט װײ." + "Arabic(2): أنا قادر على أكل الزجاج و هذا لا يؤلمني." + "Japanese: 私はガラスを食べられます。それは私を傷つけません。" + "Thai: ฉันกินกระจกได้ แต่มันไม่ทำให้ฉันเจ็บ " + ]; + + testDecoding = it "checks decoding of UTF-8 strings against Rust's String" + (builtins.map + (s: assertEq "Decoding of “${s}” is correct" (utf8.decode s) (rustDecode s)) + (lib.flatten [ + glassSentences + randomUnicode + ])); + + testDecodingEncoding = it "checks that decoding and then encoding forms an identity" + (builtins.map + (s: assertEq "Decoding and then encoding “${s}” yields itself" + (utf8.encode (utf8.decode s)) s) + (lib.flatten [ + glassSentences + randomUnicode + ])); + +in + runTestsuite "nix.utf8" [ + testFailures + testAscii + testDecoding + testDecodingEncoding + ] diff --git a/users/sterni/nixpkgs-crate-holes/default.nix b/users/sterni/nixpkgs-crate-holes/default.nix new file mode 100644 index 000000000000..a022568dc941 --- /dev/null +++ b/users/sterni/nixpkgs-crate-holes/default.nix @@ -0,0 +1,284 @@ +{ depot, pkgs, lib, ... }: + +let + # dependency imports + + inherit (depot.nix) getBins; + inherit (depot.third_party) rustsec-advisory-db; + + bins = getBins pkgs.jq [ + "jq" + ] // getBins pkgs.coreutils [ + "cat" + "printf" + "tee" + "test" + "wc" + ] // getBins pkgs.gnugrep [ + "grep" + ] // getBins pkgs.cargo-audit [ + "cargo-audit" + ] // getBins pkgs.ansi2html [ + "ansi2html" + ] // { + eprintf = depot.tools.eprintf; + }; + + # list of maintainers we may @mention on GitHub + maintainerWhitelist = builtins.attrValues { + inherit (lib.maintainers) + sternenseemann + qyliss + jk + symphorien + erictapen + expipiplus1 + ; + }; + + # buildRustPackage handling + + /* Predicate by which we identify rust packages we are interested in, + i. e. built using `buildRustPackage`. + + Type :: drv -> bool + */ + isRustPackage = v: v ? cargoDeps; + + /* Takes a buildRustPackage derivation and returns a derivation which + builds extracts the `Cargo.lock` of its `cargoDeps` derivation or + `null` if it has none. + + Type: drv -> option<drv> + */ + # TODO(sterni): support cargoVendorDir? + extractCargoLock = drv: + if !(drv ? cargoDeps.outPath) + then null + else pkgs.runCommandNoCC "${drv.name}-Cargo.lock" {} '' + if test -d "${drv.cargoDeps}"; then + cp "${drv.cargoDeps}/Cargo.lock" "$out" + fi + + if test -f "${drv.cargoDeps}"; then + tar -xO \ + --no-wildcards-match-slash --wildcards \ + -f "${drv.cargoDeps}" \ + '*/Cargo.lock' \ + > "$out" + fi + ''; + + # nixpkgs traversal + + # Condition for us to recurse: Either at top-level or recurseForDerivation. + recurseInto = path: x: path == [] || + (lib.isAttrs x && (x.recurseForDerivations or false)); + + # Returns the value or false if an eval error occurs. + tryEvalOrFalse = v: (builtins.tryEval v).value; + + /* Traverses nixpkgs as instructed by `recurseInto` and collects + the attribute and lockfile derivation of every rust package it + encounters into a list. + + Type :: attrs + -> list { + attr :: list<str>; + lock :: option<drv>; + maintainers :: list<maintainer>; + } + */ + allLockFiles = + let + go = path: x: + let + isDrv = tryEvalOrFalse (lib.isDerivation x); + doRec = tryEvalOrFalse (recurseInto path x); + isRust = tryEvalOrFalse (isRustPackage x); + in + if doRec then lib.concatLists ( + lib.mapAttrsToList (n: go (path ++ [ n ])) x + ) else if isDrv && isRust then [ + { + attr = path; + lock = extractCargoLock x; + maintainers = x.meta.maintainers or []; + } + ] else []; + in go []; + + # Report generation and formatting + + reportFor = { attr, lock, maintainers ? [] }: let + # naïve attribute path to Nix syntax conversion + strAttr = lib.concatStringsSep "." attr; + strMaintainers = lib.concatMapStringsSep " " (m: "@${m.github}") ( + builtins.filter (x: builtins.elem x maintainerWhitelist) maintainers + ); + in + if lock == null + then pkgs.emptyFile + else depot.nix.runExecline "${strAttr}-vulnerability-report" {} [ + "pipeline" [ + bins.cargo-audit + "audit" "--json" + "-n" "--db" rustsec-advisory-db + "-f" lock + ] + "importas" "out" "out" + "redirfd" "-w" "1" "$out" + bins.jq "-rj" "-f" ./format-audit-result.jq + "--arg" "attr" strAttr + "--arg" "maintainers" strMaintainers + ]; + + # GHMF in issues splits paragraphs on newlines + description = lib.concatMapStringsSep "\n\n" ( + builtins.replaceStrings [ "\n" ] [ " " ] + ) [ + '' + The vulnerability report below was generated by + [nixpkgs-crate-holes](https://code.tvl.fyi/tree/users/sterni/nixpkgs-crate-holes) + which extracts the `Cargo.lock` file of each package in nixpkgs with a + `cargoDeps` attribute and passes it to + [cargo-audit](https://github.com/RustSec/rustsec/tree/main/cargo-audit) + using RustSec's + [advisory-db at ${builtins.substring 0 7 rustsec-advisory-db.rev}](https://github.com/RustSec/advisory-db/tree/${rustsec-advisory-db.rev}/). + '' + '' + Feel free to report any problems or suggest improvements (I have an email + address on my profile and hang out on Matrix/libera.chat as sterni)! + Tick off any reports that have been fixed in the meantime. + '' + '' + Note: A vulnerability in a dependency does not necessarily mean the dependent + package is vulnerable, e. g. when a vulnerable function isn't used. + '' + ]; + + runInstructions = '' + <details> + <summary> + Generating Cargo.lock vulnerability reports + + </summary> + + If you have a checkout of [depot](https://code.tvl.fyi/about/), you can generate this report using: + + ``` + nix-build -A users.sterni.nixpkgs-crate-holes.full \ + --argstr nixpkgsPath /path/to/nixpkgs + ``` + + If you want a more detailed report for a single attribute of nixpkgs, use: + + ``` + nix-build -A users.sterni.nixpkgs-crate-holes.single \ + --argstr nixpkgsPath /path/to/nixpkgs --arg attr '[ "ripgrep" ]' + ``` + + </details> + ''; + + defaultNixpkgsArgs = { allowBroken = false; }; + + reportForNixpkgs = + { nixpkgsPath + , nixpkgsArgs ? defaultNixpkgsArgs + }@args: + + let + reports = builtins.map reportFor ( + allLockFiles (import nixpkgsPath nixpkgsArgs) + ); + in + + depot.nix.runExecline "nixpkgs-rust-pkgs-vulnerability-report.md" { + stdin = lib.concatMapStrings (report: "${report}\n") reports; + } [ + "importas" "out" "out" + "redirfd" "-w" "1" "$out" + # Print introduction paragraph for the issue + "if" [ bins.printf "%s\n\n" description ] + # Print all reports + "foreground" [ + "forstdin" "-E" "report" bins.cat "$report" + ] + # Print stats at the end (mostly as a gimmick), we already know how many + # attributes there are and count the attributes with vulnerability by + # finding the number of checkable list entries in the output. + "backtick" "-E" "vulnerableCount" [ + "pipeline" [ + bins.grep "^- \\[ \\]" "$out" + ] + bins.wc "-l" + ] + "if" [ + bins.printf + "\n%s of %s checked attributes have vulnerable dependencies.\n\n" + "$vulnerableCount" + (toString (builtins.length reports)) + ] + "if" [ + bins.printf "%s\n\n" runInstructions + ] + ]; + + singleReport = + { # Attribute to check: string or list of strings (attr path) + attr + # Path to importable nixpkgs checkout + , nixpkgsPath + # Arguments to pass to nixpkgs + , nixpkgsArgs ? defaultNixpkgsArgs + }: + + let + attr' = if builtins.isString attr then [ attr ] else attr; + drv = lib.getAttrFromPath attr' (import nixpkgsPath nixpkgsArgs); + lockFile = extractCargoLock drv; + strAttr = lib.concatStringsSep "." attr'; + in + + depot.nix.runExecline "${strAttr}-report.html" {} [ + "importas" "out" "out" + "backtick" "-I" "-E" "-N" "report" [ + bins.cargo-audit "audit" + "--quiet" + "-n" "--db" rustsec-advisory-db + "-f" lockFile + ] + "pipeline" [ + "ifte" [ + bins.printf "%s" "$report" + ] [ + bins.printf "%s\n" "No vulnerabilities found" + ] + bins.test "-n" "$report" + ] + "pipeline" [ + bins.tee "/dev/stderr" + ] + "redirfd" "-w" "1" "$out" + bins.ansi2html + ]; + +in { + full = reportForNixpkgs; + single = singleReport; + + inherit + extractCargoLock + allLockFiles + ; + + # simple sanity check, doesn't cover everything, but testing the full report + # is quite expensive in terms of evaluation. + testSingle = singleReport { + nixpkgsPath = depot.third_party.nixpkgs.path; + attr = [ "ripgrep" ]; + }; + + meta.targets = [ "testSingle" ]; +} diff --git a/users/sterni/nixpkgs-crate-holes/format-audit-result.jq b/users/sterni/nixpkgs-crate-holes/format-audit-result.jq new file mode 100644 index 000000000000..e3147b8016c1 --- /dev/null +++ b/users/sterni/nixpkgs-crate-holes/format-audit-result.jq @@ -0,0 +1,61 @@ +# Link to human-readable advisory info for a given vulnerability +def link: + [ "https://rustsec.org/advisories/", .advisory.id, ".html" ] | add; + +# Format a list of version constraints +def version_list: + [ .[] | "`" + . + "`" ] | join("; "); + +# show paths to fixing this vulnerability: +# +# - if there are patched releases, show them (the version we are using presumably +# predates the vulnerability discovery, so we likely want to upgrade to a +# patched release). +# - if there are no patched releases, show the unaffected versions (in case we +# want to downgrade). +# - otherwise we state that no unaffected versions are available at this time. +# +# This logic should be useful, but is slightly dumber than cargo-audit's +# suggestion when using the non-JSON output. +def patched: + if .versions.patched == [] then + if .versions.unaffected != [] then + "unaffected: " + (.versions.unaffected | version_list) + else + "no unaffected version available" + end + else + "patched: " + (.versions.patched | version_list) + end; + +# if the vulnerability has aliases (like CVE-*) emit them in parens +def aliases: + if .advisory.aliases == [] then + "" + else + [ " (", (.advisory.aliases | join(", ")), ")" ] | add + end; + +# each vulnerability is rendered as a (normal) sublist item +def format_vulnerability: + [ " - " + , .package.name, " ", .package.version, ": " + , "[", .advisory.id, "](", link, ")" + , aliases + , ", ", patched + , "\n" + ] | add; + +# be quiet if no found vulnerabilities, otherwise render a GHFM checklist item +if .vulnerabilities.found | not then + "" +else + ([ "- [ ] " + , "`", $attr, "`: " + , (.vulnerabilities.count | tostring) + , " vulnerabilities in Cargo.lock" + , if $maintainers != "" then " (cc " + $maintainers + ")" else "" end + , "\n" + ] + (.vulnerabilities.list | map(format_vulnerability)) + ) | add +end |