refactor(nix/buildLisp): prepare multi implementation support r/2767

Concept is roughly:

* receive extra argument `implementation` that refers to the name of an
  implementation or rather an attribute in an internal attribute set
  telling buildLisp how to do certain build steps.

* We assume an implementation can execute lisp files as scripts and that
  we can implement the following main tasks in lisp:

  - Building a library (`genCompileLisp`)

  - Building an executable (`genDumpLisp`)

  - Loading a library dynamically (`genLoadLisp`)

  Based on that we can implement:

  - Running a test suite (`genTestLisp`)

  - A REPL preloaded with a libraries and their dependencies (`lispWith`)

  Additional attributes for implementing these parts genericly  are
  added as needed (`faslExt` and `runScript`).

* `genCompileLisp` no longer prints a shell script which concatenates
  the individual FASLs. Instead it does the step previously done by the
  shell script itself. In essence `genCompileLisp` now writes a lisp
  script which compiles and installs the library to build.
  This will allow us extra freedom for different implementations, e. g.
  for ECL we'll want to build a object file archive additionally to fasl
  files in order to be able to link proper executables.

* `genLoadLisp` and `genTestLisp` are almost generic (the former just
  sometimes would need to use different file extensions), but we
  integrate them into the implementation “API” to facilitate minor
  tweaks we need to do like the `fasc` extension for ECL's native FASL
  files.

Change-Id: I1b8ccc0063159638ec7af534e9a6b5384e750193
Reviewed-on: https://cl.tvl.fyi/c/depot/+/3292
Tested-by: BuildkiteCI
Reviewed-by: tazjin <mail@tazj.in>

1 files changed, 188 insertions, 105 deletions

diff --git a/nix/buildLisp/default.nix b/nix/buildLisp/default.nix
index 3d0c36958f02..b1d852fbb51d 100644
--- a/nix/buildLisp/default.nix
+++ b/nix/buildLisp/default.nix
@@ -14,57 +14,20 @@ let
   # Internal helper definitions
   #
 
-  # 'genLoadLisp' generates Lisp code that instructs SBCL to load all
-  # the provided Lisp libraries.
-  genLoadLisp = deps: lib.concatStringsSep "\n"
-    (map (lib: "(load \"${lib}/${lib.lispName}.fasl\")") (allDeps deps));
-
-  # 'genCompileLisp' generates a Lisp file that instructs SBCL to
-  # compile the provided list of Lisp source files to $out.
-  genCompileLisp = srcs: deps: writeText "compile.lisp" ''
-    ;; This file compiles the specified sources into the Nix build
-    ;; directory, creating one FASL file for each source.
-    (require 'sb-posix)
-
-    ${genLoadLisp deps}
-
-    (defun nix-compile-lisp (file srcfile)
-      (let ((outfile (make-pathname :type "fasl"
-                                    :directory (or (sb-posix:getenv "NIX_BUILD_TOP")
-                                                   (error "not running in a Nix build"))
-                                    :name (substitute #\- #\/ srcfile))))
-        (multiple-value-bind (_outfile _warnings-p failure-p)
-            (compile-file srcfile :output-file outfile)
-          (if failure-p (sb-posix:exit 1)
-              (progn
-                ;; For the case of multiple files belonging to the same
-                ;; library being compiled, load them in order:
-                (load outfile)
-
-                ;; Write them to the FASL list in the same order:
-                (format file "cat ~a~%" (namestring outfile)))))))
-
-    (let ((*compile-verbose* t)
-          ;; FASL files are compiled into the working directory of the
-          ;; build and *then* moved to the correct out location.
-          (pwd (sb-posix:getcwd)))
-
-      (with-open-file (file "cat_fasls"
-                            :direction :output
-                            :if-does-not-exist :create)
-
-        ;; These forms were inserted by the Nix build:
-        ${
-          lib.concatStringsSep "\n" (map (src: "(nix-compile-lisp file \"${src}\")") srcs)
-        }
-        ))
-  '';
-
-  # 'genTestLisp' generates a Lisp file that loads all sources and deps and
-  # executes expression
-  genTestLisp = name: srcs: deps: expression: writeText "${name}.lisp" ''
+  defaultImplementation = "sbcl";
+
+  # Generates lisp code which instructs the given lisp implementation to load
+  # all the given dependencies.
+  genLoadLispGeneric = impl: deps:
+    lib.concatStringsSep "\n"
+      (map (lib: "(load \"${lib}/${lib.lispName}.${impl.faslExt}\")")
+        (allDeps impl deps));
+
+  # 'genTestLispGeneric' generates a Lisp file that loads all sources and deps
+  # and executes expression for a given implementation description.
+  genTestLispGeneric = impl: { name, srcs, deps, expression }: writeText "${name}.lisp" ''
     ;; Dependencies
-    ${genLoadLisp deps}
+    ${impl.genLoadLisp deps}
 
     ;; Sources
     ${lib.concatStringsSep "\n" (map (src: "(load \"${src}\")") srcs)}
@@ -78,9 +41,16 @@ let
   dependsOn = a: b: builtins.elem a b.lispDeps;
 
   # 'allDeps' flattens the list of dependencies (and their
-  # dependencies) into one ordered list of unique deps.
-  allDeps = deps: (lib.toposort dependsOn (lib.unique (
-    lib.flatten (deps ++ (map (d: d.lispDeps) deps))
+  # dependencies) into one ordered list of unique deps which
+  # all use the given implementation.
+  allDeps = impl: deps: let
+    # The override _should_ propagate itself recursively, as every derivation
+    # would only expose its actually used dependencies
+    deps' = builtins.map (dep: dep.overrideLisp or (lib.const dep) (_: {
+      implementation = impl.name;
+    })) deps;
+  in (lib.toposort dependsOn (lib.unique (
+    lib.flatten (deps' ++ (map (d: d.lispDeps) deps'))
   ))).result;
 
   # 'allNative' extracts all native dependencies of a dependency list
@@ -90,26 +60,6 @@ let
     lib.flatten (native ++ (map (d: d.lispNativeDeps) deps))
   );
 
-  # 'genDumpLisp' generates a Lisp file that instructs SBCL to dump
-  # the currently loaded image as an executable to $out/bin/$name.
-  #
-  # TODO(tazjin): Compression is currently unsupported because the
-  # SBCL in nixpkgs is, by default, not compiled with zlib support.
-  genDumpLisp = name: main: deps: writeText "dump.lisp" ''
-    (require 'sb-posix)
-
-    ${genLoadLisp deps}
-
-    (let* ((bindir (concatenate 'string (sb-posix:getenv "out") "/bin"))
-           (outpath (make-pathname :name "${name}"
-                                   :directory bindir)))
-      (save-lisp-and-die outpath
-                         :executable t
-                         :toplevel (function ${main})
-                         :purify t))
-    ;;
-  '';
-
   # Add an `overrideLisp` attribute to a function result that works
   # similar to `overrideAttrs`, but is used specifically for the
   # arguments passed to Lisp builders.
@@ -119,44 +69,176 @@ let
 
   # 'testSuite' builds a Common Lisp test suite that loads all of srcs and deps,
   # and then executes expression to check its result
-  testSuite = { name, expression, srcs, deps ? [], native ? [] }:
+  testSuite = { name, expression, srcs, deps ? [], native ? [], impl }:
     let
       lispNativeDeps = allNative native deps;
-      lispDeps = allDeps deps;
+      lispDeps = allDeps impl deps;
     in runCommandNoCC name {
       LD_LIBRARY_PATH = lib.makeLibraryPath lispNativeDeps;
       LANG = "C.UTF-8";
     } ''
       echo "Running test suite ${name}"
 
-      ${sbcl}/bin/sbcl --script ${genTestLisp name srcs deps expression} \
-        | tee $out
+      ${impl.runScript} ${
+        impl.genTestLisp {
+          inherit name srcs deps expression;
+        }
+      } | tee $out
 
       echo "Test suite ${name} succeeded"
     '';
 
+  # 'impls' is an attribute set of attribute sets which describe how to do common
+  # tasks when building for different Common Lisp implementations. Each
+  # implementation set has the following members:
+  #
+  # Required members:
+  #
+  # - runScript :: string
+  #   Describes how to invoke the implementation from the shell, so it runs a
+  #   lisp file as a script and exits.
+  # - faslExt :: string
+  #   File extension of the implementations loadable (FASL) files.
+  #   Implementations are free to generate native object files, but with the way
+  #   buildLisp works it is required that we can also 'load' libraries, so
+  #   (additionally) building a FASL or equivalent is required.
+  # - genLoadLisp :: [ dependency ] -> string
+  #   Returns lisp code to 'load' the given dependencies. 'genLoadLispGeneric'
+  #   should work for most dependencies.
+  # - genCompileLisp :: { name, srcs, deps } -> file
+  #   Builds a lisp file which instructs the implementation to build a library
+  #   from the given source files when executed. After running at least
+  #   the file "$out/${name}.${impls.${implementation}.faslExt}" should have
+  #   been created.
+  # - genDumpLisp :: { name, main, deps } -> file
+  #   Builds a lisp file which instructs the implementation to build an
+  #   executable which runs 'main' (and exits) where 'main' is available from
+  #   'deps'. The executable should be created as "$out/bin/${name}", usually
+  #   by dumping the lisp image with the replaced toplevel function replaced.
+  # - genTestLisp :: { name, srcs, deps, expression } -> file
+  #   Builds a lisp file which loads the given 'deps' and 'srcs' files and
+  #   then evaluates 'expression'. Depending on whether 'expression' returns
+  #   true or false, the script must exit with a zero or non-zero exit code.
+  #   'genTestLispGeneric' will work for most implementations.
+  # - lispWith :: [ dependency ] -> drv
+  #   Builds a script (or dumped image) which when executed loads (or has
+  #   loaded) all given dependencies. When built this should create an executable
+  #   at "$out/bin/${implementation}".
+  impls = lib.mapAttrs (name: v: { inherit name; } // v) {
+    sbcl = {
+      runScript = "${sbcl}/bin/sbcl --script";
+      faslExt = "fasl";
+
+      # 'genLoadLisp' generates Lisp code that instructs SBCL to load all
+      # the provided Lisp libraries.
+      genLoadLisp = genLoadLispGeneric impls.sbcl;
+
+      # 'genCompileLisp' generates a Lisp file that instructs SBCL to
+      # compile the provided list of Lisp source files to "$out/${name}.fasl".
+      genCompileLisp = { name, srcs, deps }: writeText "sbcl-compile.lisp" ''
+        ;; This file compiles the specified sources into the Nix build
+        ;; directory, creating one FASL file for each source.
+        (require 'sb-posix)
+
+        ${impls.sbcl.genLoadLisp deps}
+
+        (defun nix-compile-lisp (srcfile)
+          (let ((outfile (make-pathname :type "fasl"
+                                        :directory (or (sb-posix:getenv "NIX_BUILD_TOP")
+                                                       (error "not running in a Nix build"))
+                                        :name (substitute #\- #\/ srcfile))))
+            (multiple-value-bind (out-truename _warnings-p failure-p)
+                (compile-file srcfile :output-file outfile)
+              (if failure-p (sb-posix:exit 1)
+                  (progn
+                    ;; For the case of multiple files belonging to the same
+                    ;; library being compiled, load them in order:
+                    (load out-truename)
+
+                    ;; Return pathname as a string for cat-ting it later
+                    (namestring out-truename))))))
+
+        (let ((*compile-verbose* t)
+              (catted-fasl (make-pathname :type "fasl"
+                                          :directory (or (sb-posix:getenv "out")
+                                                         (error "not running in a Nix build"))
+                                          :name "${name}")))
+
+          (with-open-file (file catted-fasl
+                                :direction :output
+                                :if-does-not-exist :create)
+
+            ;; SBCL's FASL files can just be bundled together using cat
+            (sb-ext:run-program "cat"
+             (mapcar #'nix-compile-lisp
+              ;; These forms were inserted by the Nix build:
+              '(${
+                lib.concatMapStringsSep "\n" (src: "\"${src}\"") srcs
+              }))
+             :output file :search t)))
+      '';
+
+      # 'genDumpLisp' generates a Lisp file that instructs SBCL to dump
+      # the currently loaded image as an executable to $out/bin/$name.
+      #
+      # TODO(tazjin): Compression is currently unsupported because the
+      # SBCL in nixpkgs is, by default, not compiled with zlib support.
+      genDumpLisp = { name, main, deps }: writeText "sbcl-dump.lisp" ''
+        (require 'sb-posix)
+
+        ${impls.sbcl.genLoadLisp deps}
+
+        (let* ((bindir (concatenate 'string (sb-posix:getenv "out") "/bin"))
+               (outpath (make-pathname :name "${name}"
+                                       :directory bindir)))
+          (save-lisp-and-die outpath
+                             :executable t
+                             :toplevel (function ${main})
+                             :purify t))
+      '';
+
+      genTestLisp = genTestLispGeneric impls.sbcl;
+
+      lispWith = deps:
+        let lispDeps = filter (d: !d.lispBinary) (allDeps impls.sbcl deps);
+        in writeShellScriptBin "sbcl" ''
+          export LD_LIBRARY_PATH="${lib.makeLibraryPath (allNative [] lispDeps)}"
+          export LANG="C.UTF-8"
+          exec ${sbcl}/bin/sbcl ${
+            lib.optionalString (deps != [])
+              "--load ${writeText "load.lisp" (impls.sbcl.genLoadLisp lispDeps)}"
+          } $@
+        '';
+    };
+  };
+
   #
   # Public API functions
   #
 
-  # 'library' builds a list of Common Lisp files into a single FASL
-  # which can then be loaded into SBCL.
+  # 'library' builds a list of Common Lisp files into an implementation
+  # specific library format, usually a single FASL file, which can then be
+  # loaded and built into an executable via 'program'.
   library =
     { name
+    , implementation ? defaultImplementation
     , srcs
     , deps ? []
     , native ? []
     , tests ? null
     }:
     let
+      impl = impls."${implementation}" or
+        (builtins.throw "Unkown Common Lisp Implementation ${implementation}");
       lispNativeDeps = (allNative native deps);
-      lispDeps = allDeps deps;
+      lispDeps = allDeps impl deps;
       testDrv = if ! isNull tests
         then testSuite {
           name = tests.name or "${name}-test";
           srcs = srcs ++ (tests.srcs or []);
           deps = deps ++ (tests.deps or []);
           expression = tests.expression;
+          inherit impl;
         }
         else null;
     in lib.fix (self: runCommandNoCC "${name}-cllib" {
@@ -167,28 +249,28 @@ let
         lispName = name;
         lispBinary = false;
         tests = testDrv;
-        sbcl = sbclWith [ self ];
+        ${implementation} = impl.lispWith [ self ];
       };
     } ''
       ${if ! isNull testDrv
         then "echo 'Test ${testDrv} succeeded'"
         else "echo 'No tests run'"}
-      ${sbcl}/bin/sbcl --script ${genCompileLisp srcs lispDeps}
-
-      echo "Compilation finished, assembling FASL files"
 
-      # FASL files can be combined by simply concatenating them
-      # together, but it needs to be in the compilation order.
       mkdir $out
 
-      chmod +x cat_fasls
-      ./cat_fasls > $out/${name}.fasl
+      ${impl.runScript} ${
+        impl.genCompileLisp {
+          inherit name srcs;
+          deps = lispDeps;
+        }
+      }
     '');
 
-  # 'program' creates an executable containing a dumped image of the
+  # 'program' creates an executable, usually containing a dumped image of the
   # specified sources and dependencies.
   program =
     { name
+    , implementation ? defaultImplementation
     , main ? "${name}:main"
     , srcs
     , deps ? []
@@ -196,9 +278,12 @@ let
     , tests ? null
     }:
     let
-      lispDeps = allDeps deps;
+      impl = impls."${implementation}" or
+        (builtins.throw "Unkown Common Lisp Implementation ${implementation}");
+      lispDeps = allDeps impl deps;
       libPath = lib.makeLibraryPath (allNative native lispDeps);
-      selfLib = library {
+      # overriding is used internally to propagate the implementation to use
+      selfLib = (makeOverridable library) {
         inherit name srcs native;
         deps = lispDeps;
       };
@@ -210,6 +295,7 @@ let
               srcs ++ (tests.srcs or []));
           deps = deps ++ (tests.deps or []);
           expression = tests.expression;
+          inherit impl;
         }
         else null;
     in lib.fix (self: runCommandNoCC "${name}" {
@@ -222,7 +308,7 @@ let
         lispNativeDeps = native;
         lispBinary = true;
         tests = testDrv;
-        sbcl = sbclWith [ self ];
+        ${implementation} = impl.lispWith [ self ];
       };
     } ''
       ${if ! isNull testDrv
@@ -230,8 +316,11 @@ let
         else ""}
       mkdir -p $out/bin
 
-      ${sbcl}/bin/sbcl --script ${
-        genDumpLisp name main ([ selfLib ] ++ lispDeps)
+      ${impl.runScript} ${
+        impl.genDumpLisp {
+          inherit name main;
+          deps = ([ selfLib ] ++ lispDeps);
+        }
       }
 
       wrapProgram $out/bin/${name} --prefix LD_LIBRARY_PATH : "${libPath}"
@@ -243,18 +332,12 @@ let
     inherit name;
     srcs = lib.singleton (builtins.toFile "${name}.lisp" "(require '${name})");
   };
-
-  # 'sbclWith' creates an image with the specified libraries /
-  # programs loaded.
-  sbclWith = deps:
-  let lispDeps = filter (d: !d.lispBinary) (allDeps deps);
-  in writeShellScriptBin "sbcl" ''
-    export LD_LIBRARY_PATH="${lib.makeLibraryPath (allNative [] lispDeps)}"
-    export LANG="C.UTF-8"
-    exec ${sbcl}/bin/sbcl ${lib.optionalString (deps != []) "--load ${writeText "load.lisp" (genLoadLisp lispDeps)}"} $@
-  '';
 in {
   library = makeOverridable library;
   program = makeOverridable program;
-  inherit sbclWith bundled;
+  inherit bundled;
+
+  # 'sbclWith' creates an image with the specified libraries /
+  # programs loaded in SBCL.
+  sbclWith = impls.sbcl.lispWith;
 }