diff options
Diffstat (limited to 'nix/buildLisp')
| -rw-r--r-- | nix/buildLisp/README.md | 254 | ||||
| -rw-r--r-- | nix/buildLisp/default.nix | 779 | ||||
| -rw-r--r-- | nix/buildLisp/example/default.nix | 33 | ||||
| -rw-r--r-- | nix/buildLisp/example/lib.lisp | 6 | ||||
| -rw-r--r-- | nix/buildLisp/example/main.lisp | 7 | ||||
| -rw-r--r-- | nix/buildLisp/tests/argv0.nix | 36 |
6 files changed, 1115 insertions, 0 deletions
diff --git a/nix/buildLisp/README.md b/nix/buildLisp/README.md new file mode 100644 index 000000000000..0d1e46983422 --- /dev/null +++ b/nix/buildLisp/README.md @@ -0,0 +1,254 @@ +buildLisp.nix +============= + +This is a build system for Common Lisp, written in Nix. + +It aims to offer an alternative to ASDF for users who live in a +Nix-based ecosystem. This offers several advantages over ASDF: + +* Simpler (almost logic-less) package definitions +* Easy linking of native dependencies (from Nix) +* Composability with Nix tooling for other languages +* Effective, per-system caching strategies +* Easy overriding of dependencies and whatnot +* Convenient support for multiple Common Lisp implementations +* ... and more! + +The project is still in its early stages and some important +restrictions should be highlighted: + +* Extending `buildLisp` with support for a custom implementation + currently requires some knowledge of internals and may not be + considered stable yet. +* Parallel compilation is not possible: Since buildLisp doesn't encode + dependencies between components (i. e. source files) like ASDF, + it must compile source files in sequence to avoid errors due to + undefined symbols. + +## Usage + +`buildLisp` exposes four different functions: + +* `buildLisp.library`: Builds a collection of Lisp files into a library. + + | parameter | type | use | required? | + |-----------|--------------|-------------------------------|-----------| + | `name` | `string` | Name of the library | yes | + | `srcs` | `list<path>` | List of paths to source files | yes | + | `deps` | `list<drv>` | List of dependencies | no | + | `native` | `list<drv>` | List of native dependencies | no | + | `test` | see "Tests" | Specification for test suite | no | + | `implementation` | see "Implementations" | Common Lisp implementation to use | no | + + The output of invoking this is a directory containing a FASL file + that is the concatenated result of all compiled sources. + +* `buildLisp.program`: Builds an executable program out of Lisp files. + + | parameter | type | use | required? | + |-----------|--------------|-------------------------------|-----------| + | `name` | `string` | Name of the program | yes | + | `srcs` | `list<path>` | List of paths to source files | yes | + | `deps` | `list<drv>` | List of dependencies | no | + | `native` | `list<drv>` | List of native dependencies | no | + | `main` | `string` | Entrypoint function | no | + | `test` | see "Tests" | Specification for test suite | no | + | `implementation` | see "Implementations" | Common Lisp implementation to use | no | + + The `main` parameter should be the name of a function and defaults + to `${name}:main` (i.e. the *exported* `main` function of the + package named after the program). + + The output of invoking this is a directory containing a + `bin/${name}`. + +* `buildLisp.bundled`: Creates a virtual dependency on a built-in library. + + Certain libraries ship with Lisp implementations, for example + UIOP/ASDF are commonly included but many implementations also ship + internals (such as SBCLs various `sb-*` libraries). + + This function takes a single string argument that is the name of a + built-in library and returns a "package" that simply requires this + library. + +## Tests + +Both `buildLisp.library` and `buildLisp.program` take an optional argument +`tests`, which has the following supported fields: + + | parameter | type | use | required? | + |--------------|--------------|-------------------------------|-----------| + | `name` | `string` | Name of the test suite | no | + | `expression` | `string` | Lisp expression to run tests | yes | + | `srcs` | `list<path>` | List of paths to source files | no | + | `native` | `list<drv>` | List of native dependencies | no | + +the `expression` parameter should be a Lisp expression and will be evaluated +after loading all sources and dependencies (including library/program +dependencies). It must return a non-`NIL` value if the test suite has passed. + +## Example + +Using buildLisp could look like this: + +```nix +{ buildLisp, lispPkgs }: + +let libExample = buildLisp.library { + name = "lib-example"; + srcs = [ ./lib.lisp ]; + + deps = with lispPkgs; [ + (buildLisp.bundled "sb-posix") + iterate + cl-ppcre + ]; +}; +in buildLisp.program { + name = "example"; + deps = [ libExample ]; + srcs = [ ./main.lisp ]; + tests = { + deps = [ lispPkgs.fiveam ]; + srcs = [ ./tests.lisp ]; + expression = "(fiveam:run!)"; + }; +} +``` + +## Development REPLs + +`buildLisp` builds loadable variants of both `program` and `library` derivations +(usually FASL files). Therefore it can provide a convenient way to obtain an +instance of any implementation preloaded with `buildLisp`-derivations. This +is especially useful to use as a host for Sly or SLIME. + +* `buildLisp.sbcl.lispWith`, `buildLisp.ccl.lispWith`, ...: + Creates a wrapper script preloading a Lisp implementation with various dependencies. + + This function takes a single argument which is a list of Lisp + libraries programs or programs. The desired Lisp implementation + will load all given derivations and all their dependencies on + startup. + + The shortcut `buildLisp.sbclWith` for `buildLisp.sbcl.lispWith` is also provided. + +* `repl` passthru attribute: `derivation.repl` is provided as a shortcut + for `buildLisp.${implementationName}.lispWith [ derivation ]`. + `derivation.ccl.repl`, `derivation.sbcl.repl` etc. work as well, of course + (see also "Implementations" section). + +## Implementations + +Both `buildLisp.library` and `buildLisp.program` allow specifying a different +Common Lisp implementation than the default one (which is SBCL). When an +implementation is passed, `buildLisp` makes sure all dependencies are built +with that implementation as well since build artifacts from different +implementation will be incompatible with each other. + +The argument taken by `implementation` is a special attribute set which +describes how to do certain tasks for a given implementation, like building +or loading a library. In case you want to use a custom implementation +description, the precise structure needed is documented in `buildLisp`'s +source code for now. `buildLisp` also exposes the following already +working implementation sets: + +* `buildLisp.sbcl`: [SBCL][sbcl], our default implementation + +* `buildLisp.ccl`: [CCL][ccl], similar to SBCL, but with very good macOS support + +* `buildLisp.ecl`: [ECL][ecl] setup to produce statically linked binaries and + libraries. Note that its runtime library is LGPL, so [extra conditions][lgpl-static] + must be fulfilled when distributing binaries produced this way. + +* Support for ABCL is planned. + +For every of these “known” implementations, `buildLisp` will create a `passthru` +attribute named like the implementation which points to a variant of the derivation +built with said implementation. Say we have a derivation, `myDrv`, built using SBCL: +While `myDrv` and `myDrv.sbcl` are built using SBCL, `myDrv.ecl`, `myDrv.ccl` etc. +build the derivation and all its dependencies using ECL and CCL respectively. + +This is useful to test portability of your derivation, but is also used internally +to speed up the “normalization” of the dependency graph. Thus it is important to +make sure that your custom implementation's name doesn't clash with one of the +“known” ones. + +## Handling Implementation Specifics + +When targeting multiple Common Lisp implementation, it is often necessary to +handle differing interfaces for OS interaction or to make use of special +implementation features. For this reason, `buildLisp` allows specifying +dependencies and source files for specific implementations only. This can +be utilized by having an attribute set in the list for the `deps` or `srcs` +argument: `buildLisp` will pick the value of the attribute named like the +used implementation or `default` and ignore the set completely if both +are missing. + +```nix +{ buildLisp, lispPkgs }: + +buildLisp.library { + name = "mylib"; + + srcs = [ + # These are included always of course + ./package.lisp + ./portable-lib.lisp + + # Choose right impl-* file + { + sbcl = ./impl-sbcl.lisp; + ccl = ./impl-ccl.lisp; + ecl = ./impl-ecl.lisp; + } + + # We can also use this to inject extra files + { ecl = ./extra-ecl-optimizations.lisp; } + ]; + + deps = [ + # Use SBCL's special bundled package, flexi-streams otherwise + { + sbcl = buildLisp.bundled "sb-rotate-byte"; + default = lispPkgs.flexi-streams; + } + ]; +} +``` + +Additionally a `brokenOn` parameter is accepted which takes a list of +implementation names on which the derivation is not expected to work. +This only influences `meta.ci.targets` which is read by depot's CI to +check which variants (see "Implementations") of the derivation to +build, so it may not be useful outside of depot. + +## Influencing the Lisp Runtime + +Lisp implementations which create an executable by dumping an image +usually parse a few implementation-specific command line options on +executable startup that influence runtime settings related to things +like GC. `buildLisp` generates a wrapper which makes sure that this +never interferes with the argument parsing implemented in the actual +application, but sometimes it is useful to run an executable with +special settings. To allow this, the content of `NIX_BUILDLISP_LISP_ARGS` +is passed to the lisp implementation. + +For example, you can make the underlying SBCL print its version for +any executable built with `buildLisp` (and SBCL) like this: + +```console +$ env NIX_BUILDLISP_LISP_ARGS="--version" ./result/bin/🕰️ +SBCL 2.1.2.nixos +``` + +In practice you'd probably want to specify options like +`--dynamic-space-size` or `--tls-limit` (try passing `--help` for a +full list). Naturally, these options are completely different for +different implementations. + +[sbcl]: http://www.sbcl.org/ +[ccl]: https://ccl.clozure.com/ +[ecl]: https://common-lisp.net/project/ecl/ +[lgpl-static]: https://www.gnu.org/licenses/gpl-faq.en.html#LGPLStaticVsDynamic diff --git a/nix/buildLisp/default.nix b/nix/buildLisp/default.nix new file mode 100644 index 000000000000..a8168334a77b --- /dev/null +++ b/nix/buildLisp/default.nix @@ -0,0 +1,779 @@ +# buildLisp provides Nix functions to build Common Lisp packages, +# targeting SBCL. +# +# buildLisp is designed to enforce conventions and do away with the +# free-for-all of existing Lisp build systems. + +{ pkgs ? import <nixpkgs> { }, ... }: + +let + inherit (builtins) map elemAt match filter; + inherit (pkgs) lib runCommandNoCC makeWrapper writeText writeShellScriptBin sbcl ecl-static ccl; + inherit (pkgs.stdenv) targetPlatform; + + # + # Internal helper definitions + # + + defaultImplementation = impls.sbcl; + + # Many Common Lisp implementations (like ECL and CCL) will occasionally drop + # you into an interactive debugger even when executing something as a script. + # In nix builds we don't want such a situation: Any error should make the + # script exit non-zero. Luckily the ANSI standard specifies *debugger-hook* + # which is invoked before the debugger letting us just do that. + disableDebugger = writeText "disable-debugger.lisp" '' + (setf *debugger-hook* + (lambda (error hook) + (declare (ignore hook)) + (format *error-output* "~%Unhandled error: ~a~%" error) + #+ccl (quit 1) + #+ecl (ext:quit 1))) + ''; + + # Process a list of arbitrary values which also contains “implementation + # filter sets” which describe conditonal inclusion of elements depending + # on the CL implementation used. Elements are processed in the following + # manner: + # + # * Paths, strings, derivations are left as is + # * A non-derivation attribute set is processed like this: + # 1. If it has an attribute equal to impl.name, replace with its value. + # 2. Alternatively use the value of the "default" attribute. + # 3. In all other cases delete the element from the list. + # + # This can be used to express dependencies or source files which are specific + # to certain implementations: + # + # srcs = [ + # # mixable with unconditional entries + # ./package.lisp + # + # # implementation specific source files + # { + # ccl = ./impl-ccl.lisp; + # sbcl = ./impl-sbcl.lisp; + # ecl = ./impl-ecl.lisp; + # } + # ]; + # + # deps = [ + # # this dependency is ignored if impl.name != "sbcl" + # { sbcl = buildLisp.bundled "sb-posix"; } + # + # # only special casing for a single implementation + # { + # sbcl = buildLisp.bundled "uiop"; + # default = buildLisp.bundled "asdf"; + # } + # ]; + implFilter = impl: xs: + let + isFilterSet = x: builtins.isAttrs x && !(lib.isDerivation x); + in + builtins.map + ( + x: if isFilterSet x then x.${impl.name} or x.default else x + ) + (builtins.filter + ( + x: !(isFilterSet x) || x ? ${impl.name} || x ? default + ) + xs); + + # Generates lisp code which instructs the given lisp implementation to load + # all the given dependencies. + genLoadLispGeneric = impl: deps: + lib.concatStringsSep "\n" + (map (lib: "(load \"${lib}/${lib.lispName}.${impl.faslExt}\")") + (allDeps impl deps)); + + # 'genTestLispGeneric' generates a Lisp file that loads all sources and deps + # and executes expression for a given implementation description. + genTestLispGeneric = impl: { name, srcs, deps, expression }: writeText "${name}.lisp" '' + ;; Dependencies + ${impl.genLoadLisp deps} + + ;; Sources + ${lib.concatStringsSep "\n" (map (src: "(load \"${src}\")") srcs)} + + ;; Test expression + (unless ${expression} + (exit :code 1)) + ''; + + # 'dependsOn' determines whether Lisp library 'b' depends on 'a'. + dependsOn = a: b: builtins.elem a b.lispDeps; + + # 'allDeps' flattens the list of dependencies (and their + # dependencies) into one ordered list of unique deps which + # all use the given implementation. + allDeps = impl: deps: + let + # The override _should_ propagate itself recursively, as every derivation + # would only expose its actually used dependencies. Use implementation + # attribute created by withExtras if present, override in all other cases + # (mainly bundled). + deps' = builtins.map + (dep: dep."${impl.name}" or (dep.overrideLisp (_: { + implementation = impl; + }))) + deps; + in + (lib.toposort dependsOn (lib.unique ( + lib.flatten (deps' ++ (map (d: d.lispDeps) deps')) + ))).result; + + # 'allNative' extracts all native dependencies of a dependency list + # to ensure that library load paths are set correctly during all + # compilations and program assembly. + allNative = native: deps: lib.unique ( + lib.flatten (native ++ (map (d: d.lispNativeDeps) deps)) + ); + + # Add an `overrideLisp` attribute to a function result that works + # similar to `overrideAttrs`, but is used specifically for the + # arguments passed to Lisp builders. + makeOverridable = f: orig: (f orig) // { + overrideLisp = new: makeOverridable f (orig // (new orig)); + }; + + # This is a wrapper arround 'makeOverridable' which performs its + # function, but also adds a the following additional attributes to the + # resulting derivation, namely a repl attribute which builds a `lispWith` + # derivation for the current implementation and additional attributes for + # every all implementations. So `drv.sbcl` would build the derivation + # with SBCL regardless of what was specified in the initial arguments. + withExtras = f: args: + let + drv = (makeOverridable f) args; + in + lib.fix (self: + drv.overrideLisp + (old: + let + implementation = old.implementation or defaultImplementation; + brokenOn = old.brokenOn or [ ]; + targets = lib.subtractLists (brokenOn ++ [ implementation.name ]) + (builtins.attrNames impls); + in + { + passthru = (old.passthru or { }) // { + repl = implementation.lispWith [ self ]; + + # meta is done via passthru to minimize rebuilds caused by overriding + meta = (old.passthru.meta or { }) // { + ci = (old.passthru.meta.ci or { }) // { + inherit targets; + }; + }; + } // builtins.listToAttrs (builtins.map + (impl: { + inherit (impl) name; + value = self.overrideLisp (_: { + implementation = impl; + }); + }) + (builtins.attrValues impls)); + }) // { + overrideLisp = new: withExtras f (args // new args); + }); + + # 'testSuite' builds a Common Lisp test suite that loads all of srcs and deps, + # and then executes expression to check its result + testSuite = { name, expression, srcs, deps ? [ ], native ? [ ], implementation }: + let + lispDeps = allDeps implementation (implFilter implementation deps); + lispNativeDeps = allNative native lispDeps; + filteredSrcs = implFilter implementation srcs; + in + runCommandNoCC name + { + LD_LIBRARY_PATH = lib.makeLibraryPath lispNativeDeps; + LANG = "C.UTF-8"; + } '' + echo "Running test suite ${name}" + + ${implementation.runScript} ${ + implementation.genTestLisp { + inherit name expression; + srcs = filteredSrcs; + deps = lispDeps; + } + } | tee $out + + echo "Test suite ${name} succeeded" + ''; + + # 'impls' is an attribute set of attribute sets which describe how to do common + # tasks when building for different Common Lisp implementations. Each + # implementation set has the following members: + # + # Required members: + # + # - runScript :: string + # Describes how to invoke the implementation from the shell, so it runs a + # lisp file as a script and exits. + # - faslExt :: string + # File extension of the implementations loadable (FASL) files. + # Implementations are free to generate native object files, but with the way + # buildLisp works it is required that we can also 'load' libraries, so + # (additionally) building a FASL or equivalent is required. + # - genLoadLisp :: [ dependency ] -> string + # Returns lisp code to 'load' the given dependencies. 'genLoadLispGeneric' + # should work for most dependencies. + # - genCompileLisp :: { name, srcs, deps } -> file + # Builds a lisp file which instructs the implementation to build a library + # from the given source files when executed. After running at least + # the file "$out/${name}.${impls.${implementation}.faslExt}" should have + # been created. + # - genDumpLisp :: { name, main, deps } -> file + # Builds a lisp file which instructs the implementation to build an + # executable which runs 'main' (and exits) where 'main' is available from + # 'deps'. The executable should be created as "$out/bin/${name}", usually + # by dumping the lisp image with the replaced toplevel function replaced. + # - wrapProgram :: boolean + # Whether to wrap the resulting binary / image with a wrapper script setting + # `LD_LIBRARY_PATH`. + # - genTestLisp :: { name, srcs, deps, expression } -> file + # Builds a lisp file which loads the given 'deps' and 'srcs' files and + # then evaluates 'expression'. Depending on whether 'expression' returns + # true or false, the script must exit with a zero or non-zero exit code. + # 'genTestLispGeneric' will work for most implementations. + # - lispWith :: [ dependency ] -> drv + # Builds a script (or dumped image) which when executed loads (or has + # loaded) all given dependencies. When built this should create an executable + # at "$out/bin/${implementation}". + # + # Optional members: + # + # - bundled :: string -> library + # Allows giving an implementation specific builder for a bundled library. + # This function is used as a replacement for the internal defaultBundled + # function and only needs to support one implementation. The returned derivation + # must behave like one built by 'library' (in particular have the same files + # available in "$out" and the same 'passthru' attributes), but may be built + # completely differently. + impls = lib.mapAttrs (name: v: { inherit name; } // v) { + sbcl = { + runScript = "${sbcl}/bin/sbcl --script"; + faslExt = "fasl"; + + # 'genLoadLisp' generates Lisp code that instructs SBCL to load all + # the provided Lisp libraries. + genLoadLisp = genLoadLispGeneric impls.sbcl; + + # 'genCompileLisp' generates a Lisp file that instructs SBCL to + # compile the provided list of Lisp source files to "$out/${name}.fasl". + genCompileLisp = { name, srcs, deps }: writeText "sbcl-compile.lisp" '' + ;; This file compiles the specified sources into the Nix build + ;; directory, creating one FASL file for each source. + (require 'sb-posix) + + ${impls.sbcl.genLoadLisp deps} + + (defun nix-compile-lisp (srcfile) + (let ((outfile (make-pathname :type "fasl" + :directory (or (sb-posix:getenv "NIX_BUILD_TOP") + (error "not running in a Nix build")) + :name (substitute #\- #\/ srcfile)))) + (multiple-value-bind (out-truename _warnings-p failure-p) + (compile-file srcfile :output-file outfile) + (if failure-p (sb-posix:exit 1) + (progn + ;; For the case of multiple files belonging to the same + ;; library being compiled, load them in order: + (load out-truename) + + ;; Return pathname as a string for cat-ting it later + (namestring out-truename)))))) + + (let ((*compile-verbose* t) + (catted-fasl (make-pathname :type "fasl" + :directory (or (sb-posix:getenv "out") + (error "not running in a Nix build")) + :name "${name}"))) + + (with-open-file (file catted-fasl + :direction :output + :if-does-not-exist :create) + + ;; SBCL's FASL files can just be bundled together using cat + (sb-ext:run-program "cat" + (mapcar #'nix-compile-lisp + ;; These forms were inserted by the Nix build: + '(${ + lib.concatMapStringsSep "\n" (src: "\"${src}\"") srcs + })) + :output file :search t))) + ''; + + # 'genDumpLisp' generates a Lisp file that instructs SBCL to dump + # the currently loaded image as an executable to $out/bin/$name. + # + # TODO(tazjin): Compression is currently unsupported because the + # SBCL in nixpkgs is, by default, not compiled with zlib support. + genDumpLisp = { name, main, deps }: writeText "sbcl-dump.lisp" '' + (require 'sb-posix) + + ${impls.sbcl.genLoadLisp deps} + + (let* ((bindir (concatenate 'string (sb-posix:getenv "out") "/bin")) + (outpath (make-pathname :name "${name}" + :directory bindir))) + + ;; Tell UIOP that argv[0] will refer to running image, not the lisp impl + (when (find-package :uiop) + (eval `(setq ,(find-symbol "*IMAGE-DUMPED-P*" :uiop) :executable))) + + (save-lisp-and-die outpath + :executable t + :toplevel + (lambda () + ;; Filter out everything prior to the `--` we + ;; insert in the wrapper to prevent SBCL from + ;; parsing arguments at startup + (setf sb-ext:*posix-argv* + (delete "--" sb-ext:*posix-argv* + :test #'string= :count 1)) + (${main})) + :purify t)) + ''; + + wrapProgram = true; + + genTestLisp = genTestLispGeneric impls.sbcl; + + lispWith = deps: + let lispDeps = filter (d: !d.lispBinary) (allDeps impls.sbcl deps); + in writeShellScriptBin "sbcl" '' + export LD_LIBRARY_PATH="${lib.makeLibraryPath (allNative [] lispDeps)}" + export LANG="C.UTF-8" + exec ${sbcl}/bin/sbcl ${ + lib.optionalString (deps != []) + "--load ${writeText "load.lisp" (impls.sbcl.genLoadLisp lispDeps)}" + } $@ + ''; + }; + ecl = { + runScript = "${ecl-static}/bin/ecl --load ${disableDebugger} --shell"; + faslExt = "fasc"; + genLoadLisp = genLoadLispGeneric impls.ecl; + genCompileLisp = { name, srcs, deps }: writeText "ecl-compile.lisp" '' + ;; This seems to be required to bring make the 'c' package available + ;; early, otherwise ECL tends to fail with a read failure… + (ext:install-c-compiler) + + ;; Load dependencies + ${impls.ecl.genLoadLisp deps} + + (defun getenv-or-fail (var) + (or (ext:getenv var) + (error (format nil "Missing expected environment variable ~A" var)))) + + (defun nix-compile-file (srcfile &key native) + "Compile the given srcfile into a compilation unit in :out-dir using + a unique name based on srcfile as the filename which is returned after + compilation. If :native is true, create an native object file, + otherwise a byte-compile fasc file is built and immediately loaded." + + (let* ((unique-name (substitute #\_ #\/ srcfile)) + (out-file (make-pathname :type (if native "o" "fasc") + :directory (getenv-or-fail "NIX_BUILD_TOP") + :name unique-name))) + (multiple-value-bind (out-truename _warnings-p failure-p) + (compile-file srcfile :system-p native + :load (not native) + :output-file out-file + :verbose t :print t) + (if failure-p (ext:quit 1) out-truename)))) + + (let* ((out-dir (getenv-or-fail "out")) + (nix-build-dir (getenv-or-fail "NIX_BUILD_TOP")) + (srcs + ;; These forms are inserted by the Nix build + '(${lib.concatMapStringsSep "\n" (src: "\"${src}\"") srcs}))) + + ;; First, we'll byte compile loadable FASL files and load them + ;; immediately. Since we are using a statically linked ECL, there's + ;; no way to load native objects, so we rely on byte compilation + ;; for all our loading — which is crucial in compilation of course. + (ext:install-bytecodes-compiler) + + ;; ECL's bytecode FASLs can just be concatenated to create a bundle + ;; at least since a recent bugfix which we apply as a patch. + ;; See also: https://gitlab.com/embeddable-common-lisp/ecl/-/issues/649 + (let ((bundle-out (make-pathname :type "fasc" :name "${name}" + :directory out-dir))) + + (with-open-file (fasc-stream bundle-out :direction :output) + (ext:run-program "cat" + (mapcar (lambda (f) + (namestring + (nix-compile-file f :native nil))) + srcs) + :output fasc-stream))) + + (ext:install-c-compiler) + + ;; Build a (natively compiled) static archive (.a) file. We want to + ;; use this for (statically) linking an executable later. The bytecode + ;; dance is only required because we can't load such archives. + (c:build-static-library + (make-pathname :type "a" :name "${name}" :directory out-dir) + :lisp-files (mapcar (lambda (x) + (nix-compile-file x :native t)) + srcs))) + ''; + genDumpLisp = { name, main, deps }: writeText "ecl-dump.lisp" '' + (defun getenv-or-fail (var) + (or (ext:getenv var) + (error (format nil "Missing expected environment variable ~A" var)))) + + ${impls.ecl.genLoadLisp deps} + + ;; makes a 'c' package available that can link executables + (ext:install-c-compiler) + + (c:build-program + (merge-pathnames (make-pathname :directory '(:relative "bin") + :name "${name}") + (truename (getenv-or-fail "out"))) + :epilogue-code `(progn + ;; UIOP doesn't understand ECL, so we need to make it + ;; aware that we are a proper executable, causing it + ;; to handle argument parsing and such properly. Since + ;; this needs to work even when we're not using UIOP, + ;; we need to do some compile-time acrobatics. + ,(when (find-package :uiop) + `(setf ,(find-symbol "*IMAGE-DUMPED-P*" :uiop) :executable)) + ;; Run the actual application… + (${main}) + ;; … and exit. + (ext:quit)) + ;; ECL can't remember these from its own build… + :ld-flags '("-static") + :lisp-files + ;; The following forms are inserted by the Nix build + '(${ + lib.concatMapStrings (dep: '' + "${dep}/${dep.lispName}.a" + '') (allDeps impls.ecl deps) + })) + ''; + + wrapProgram = false; + + genTestLisp = genTestLispGeneric impls.ecl; + + lispWith = deps: + let lispDeps = filter (d: !d.lispBinary) (allDeps impls.ecl deps); + in writeShellScriptBin "ecl" '' + exec ${ecl-static}/bin/ecl ${ + lib.optionalString (deps != []) + "--load ${writeText "load.lisp" (impls.ecl.genLoadLisp lispDeps)}" + } $@ + ''; + + bundled = name: runCommandNoCC "${name}-cllib" + { + passthru = { + lispName = name; + lispNativeDeps = [ ]; + lispDeps = [ ]; + lispBinary = false; + repl = impls.ecl.lispWith [ (impls.ecl.bundled name) ]; + }; + } '' + mkdir -p "$out" + ln -s "${ecl-static}/lib/ecl-${ecl-static.version}/${name}.${impls.ecl.faslExt}" -t "$out" + ln -s "${ecl-static}/lib/ecl-${ecl-static.version}/lib${name}.a" "$out/${name}.a" + ''; + }; + ccl = { + # Relatively bespoke wrapper script necessary to make CCL just™ execute + # a lisp file as a script. + runScript = pkgs.writers.writeBash "ccl" '' + # don't print intro message etc. + args=("--quiet") + + # makes CCL crash on error instead of entering the debugger + args+=("--load" "${disableDebugger}") + + # load files from command line in order + for f in "$@"; do + args+=("--load" "$f") + done + + # Exit if everything was processed successfully + args+=("--eval" "(quit)") + + exec ${ccl}/bin/ccl ''${args[@]} + ''; + + # See https://ccl.clozure.com/docs/ccl.html#building-definitions + faslExt = + /**/ + if targetPlatform.isPowerPC && targetPlatform.is32bit then "pfsl" + else if targetPlatform.isPowerPC && targetPlatform.is64bit then "p64fsl" + else if targetPlatform.isx86_64 && targetPlatform.isLinux then "lx64fsl" + else if targetPlatform.isx86_32 && targetPlatform.isLinux then "lx32fsl" + else if targetPlatform.isAarch32 && targetPlatform.isLinux then "lafsl" + else if targetPlatform.isx86_32 && targetPlatform.isDarwin then "dx32fsl" + else if targetPlatform.isx86_64 && targetPlatform.isDarwin then "dx64fsl" + else if targetPlatform.isx86_64 && targetPlatform.isDarwin then "dx64fsl" + else if targetPlatform.isx86_32 && targetPlatform.isFreeBSD then "fx32fsl" + else if targetPlatform.isx86_64 && targetPlatform.isFreeBSD then "fx64fsl" + else if targetPlatform.isx86_32 && targetPlatform.isWindows then "wx32fsl" + else if targetPlatform.isx86_64 && targetPlatform.isWindows then "wx64fsl" + else builtins.throw "Don't know what FASLs are called for this platform: " + + pkgs.stdenv.targetPlatform.system; + + genLoadLisp = genLoadLispGeneric impls.ccl; + + genCompileLisp = { name, srcs, deps }: writeText "ccl-compile.lisp" '' + ${impls.ccl.genLoadLisp deps} + + (defun getenv-or-fail (var) + (or (getenv var) + (error (format nil "Missing expected environment variable ~A" var)))) + + (defun nix-compile-file (srcfile) + "Trivial wrapper around COMPILE-FILE which causes CCL to exit if + compilation fails and LOADs the compiled file on success." + (let ((output (make-pathname :name (substitute #\_ #\/ srcfile) + :type "${impls.ccl.faslExt}" + :directory (getenv-or-fail "NIX_BUILD_TOP")))) + (multiple-value-bind (out-truename _warnings-p failure-p) + (compile-file srcfile :output-file output :print t :verbose t) + (declare (ignore _warnings-p)) + (if failure-p (quit 1) + (progn (load out-truename) out-truename))))) + + (fasl-concatenate (make-pathname :name "${name}" :type "${impls.ccl.faslExt}" + :directory (getenv-or-fail "out")) + (mapcar #'nix-compile-file + ;; These forms where inserted by the Nix build + '(${ + lib.concatMapStrings (src: '' + "${src}" + '') srcs + }))) + ''; + + genDumpLisp = { name, main, deps }: writeText "ccl-dump.lisp" '' + ${impls.ccl.genLoadLisp deps} + + (let* ((out (or (getenv "out") (error "Not running in a Nix build"))) + (bindir (concatenate 'string out "/bin/")) + (executable (make-pathname :directory bindir :name "${name}"))) + + ;; Tell UIOP that argv[0] will refer to running image, not the lisp impl + (when (find-package :uiop) + (eval `(setf ,(find-symbol "*IMAGE-DUMPED-P*" :uiop) :executable))) + + (save-application executable + :purify t + :error-handler :quit + :toplevel-function + (lambda () + ;; Filter out everything prior to the `--` we + ;; insert in the wrapper to prevent SBCL from + ;; parsing arguments at startup + (setf ccl:*command-line-argument-list* + (delete "--" ccl:*command-line-argument-list* + :test #'string= :count 1)) + (${main})) + :mode #o755 + ;; TODO(sterni): use :native t on macOS + :prepend-kernel t)) + ''; + + wrapProgram = true; + + genTestLisp = genTestLispGeneric impls.ccl; + + lispWith = deps: + let lispDeps = filter (d: !d.lispBinary) (allDeps impls.ccl deps); + in writeShellScriptBin "ccl" '' + export LD_LIBRARY_PATH="${lib.makeLibraryPath (allNative [] lispDeps)}" + exec ${ccl}/bin/ccl ${ + lib.optionalString (deps != []) + "--load ${writeText "load.lisp" (impls.ccl.genLoadLisp lispDeps)}" + } "$@" + ''; + }; + }; + + # + # Public API functions + # + + # 'library' builds a list of Common Lisp files into an implementation + # specific library format, usually a single FASL file, which can then be + # loaded and built into an executable via 'program'. + library = + { name + , implementation ? defaultImplementation + , brokenOn ? [ ] # TODO(sterni): make this a warning + , srcs + , deps ? [ ] + , native ? [ ] + , tests ? null + , passthru ? { } + }: + let + filteredDeps = implFilter implementation deps; + filteredSrcs = implFilter implementation srcs; + lispNativeDeps = (allNative native filteredDeps); + lispDeps = allDeps implementation filteredDeps; + testDrv = + if ! isNull tests + then + testSuite + { + name = tests.name or "${name}-test"; + srcs = filteredSrcs ++ (tests.srcs or [ ]); + deps = filteredDeps ++ (tests.deps or [ ]); + expression = tests.expression; + inherit implementation; + } + else null; + in + lib.fix (self: runCommandNoCC "${name}-cllib" + { + LD_LIBRARY_PATH = lib.makeLibraryPath lispNativeDeps; + LANG = "C.UTF-8"; + passthru = passthru // { + inherit lispNativeDeps lispDeps; + lispName = name; + lispBinary = false; + tests = testDrv; + }; + } '' + ${if ! isNull testDrv + then "echo 'Test ${testDrv} succeeded'" + else "echo 'No tests run'"} + + mkdir $out + + ${implementation.runScript} ${ + implementation.genCompileLisp { + srcs = filteredSrcs; + inherit name; + deps = lispDeps; + } + } + ''); + + # 'program' creates an executable, usually containing a dumped image of the + # specified sources and dependencies. + program = + { name + , implementation ? defaultImplementation + , brokenOn ? [ ] # TODO(sterni): make this a warning + , main ? "${name}:main" + , srcs + , deps ? [ ] + , native ? [ ] + , tests ? null + , passthru ? { } + }: + let + filteredSrcs = implFilter implementation srcs; + filteredDeps = implFilter implementation deps; + lispDeps = allDeps implementation filteredDeps; + libPath = lib.makeLibraryPath (allNative native lispDeps); + # overriding is used internally to propagate the implementation to use + selfLib = (makeOverridable library) { + inherit name native brokenOn; + deps = lispDeps; + srcs = filteredSrcs; + }; + testDrv = + if ! isNull tests + then + testSuite + { + name = tests.name or "${name}-test"; + srcs = + ( + # testSuite does run implFilter as well + filteredSrcs ++ (tests.srcs or [ ]) + ); + deps = filteredDeps ++ (tests.deps or [ ]); + expression = tests.expression; + inherit implementation; + } + else null; + in + lib.fix (self: runCommandNoCC "${name}" + { + nativeBuildInputs = [ makeWrapper ]; + LD_LIBRARY_PATH = libPath; + LANG = "C.UTF-8"; + passthru = passthru // { + lispName = name; + lispDeps = [ selfLib ]; + lispNativeDeps = native; + lispBinary = true; + tests = testDrv; + }; + } + ('' + ${if ! isNull testDrv + then "echo 'Test ${testDrv} succeeded'" + else ""} + mkdir -p $out/bin + + ${implementation.runScript} ${ + implementation.genDumpLisp { + inherit name main; + deps = ([ selfLib ] ++ lispDeps); + } + } + '' + lib.optionalString implementation.wrapProgram '' + wrapProgram $out/bin/${name} \ + --prefix LD_LIBRARY_PATH : "${libPath}" \ + --add-flags "\$NIX_BUILDLISP_LISP_ARGS --" + '')); + + # 'bundled' creates a "library" which makes a built-in package available, + # such as any of SBCL's sb-* packages or ASDF. By default this is done + # by calling 'require', but implementations are free to provide their + # own specific bundled function. + bundled = name: + let + # TODO(sterni): allow overriding args to underlying 'library' (e. g. srcs) + defaultBundled = implementation: name: library { + inherit name implementation; + srcs = lib.singleton (builtins.toFile "${name}.lisp" "(require '${name})"); + }; + + bundled' = + { implementation ? defaultImplementation + , name + }: + implementation.bundled or (defaultBundled implementation) name; + + in + (makeOverridable bundled') { + inherit name; + }; + +in +{ + library = withExtras library; + program = withExtras program; + inherit bundled; + + # 'sbclWith' creates an image with the specified libraries / + # programs loaded in SBCL. + sbclWith = impls.sbcl.lispWith; + + inherit (impls) + sbcl + ecl + ccl + ; +} diff --git a/nix/buildLisp/example/default.nix b/nix/buildLisp/example/default.nix new file mode 100644 index 000000000000..6add2676f10c --- /dev/null +++ b/nix/buildLisp/example/default.nix @@ -0,0 +1,33 @@ +{ depot, ... }: + +let + inherit (depot.nix) buildLisp; + + # Example Lisp library. + # + # Currently the `name` attribute is only used for the derivation + # itself, it has no practical implications. + libExample = buildLisp.library { + name = "lib-example"; + srcs = [ + ./lib.lisp + ]; + }; + + # Example Lisp program. + # + # This builds & writes an executable for a program using the library + # above to disk. + # + # By default, buildLisp.program expects the entry point to be + # `$name:main`. This can be overridden by configuring the `main` + # attribute. +in +buildLisp.program { + name = "example"; + deps = [ libExample ]; + + srcs = [ + ./main.lisp + ]; +} diff --git a/nix/buildLisp/example/lib.lisp b/nix/buildLisp/example/lib.lisp new file mode 100644 index 000000000000..e557de4ae5fd --- /dev/null +++ b/nix/buildLisp/example/lib.lisp @@ -0,0 +1,6 @@ +(defpackage lib-example + (:use :cl) + (:export :who)) +(in-package :lib-example) + +(defun who () "edef") diff --git a/nix/buildLisp/example/main.lisp b/nix/buildLisp/example/main.lisp new file mode 100644 index 000000000000..a29390cf4dba --- /dev/null +++ b/nix/buildLisp/example/main.lisp @@ -0,0 +1,7 @@ +(defpackage example + (:use :cl :lib-example) + (:export :main)) +(in-package :example) + +(defun main () + (format t "i <3 ~A~%" (who))) diff --git a/nix/buildLisp/tests/argv0.nix b/nix/buildLisp/tests/argv0.nix new file mode 100644 index 000000000000..bc29337d06cd --- /dev/null +++ b/nix/buildLisp/tests/argv0.nix @@ -0,0 +1,36 @@ +{ depot, pkgs, ... }: + +depot.nix.buildLisp.program { + name = "argv0-test"; + + srcs = [ + (pkgs.writeText "argv0-test.lisp" '' + (defpackage :argv0-test (:use :common-lisp :uiop) (:export :main)) + (in-package :argv0-test) + + (defun main () + (format t "~A~%" (uiop:argv0))) + '') + ]; + + deps = [ + { + sbcl = depot.nix.buildLisp.bundled "uiop"; + default = depot.nix.buildLisp.bundled "asdf"; + } + ]; + + passthru.meta.ci = { + extraSteps.verify = { + label = "verify argv[0] output"; + needsOutput = true; + command = pkgs.writeShellScript "check-argv0" '' + set -eux + + for invocation in "$(pwd)/result/bin/argv0-test" "./result/bin/argv0-test"; do + test "$invocation" = "$("$invocation")" + done + ''; + }; + }; +} |