feat(nix/buildLisp): add ecl r/2771

Adds ECL as a second supported implementation, specifically a statically
linked ECL. This is interesting because we can create statically linked
binaries, but has a few drawbacks which doesn't make it generally
useful:

* Loading things is very slow: The statically linked ECL only has byte
  compilation available, so when we do load things or use the REPL it is
  significantly worse than with e. g. SBCL.

* We can't load shared objects via the FFI since ECL's dffi is not
  available when linked statically. This means that as it stands, we
  can't build a statically linked //web/panettone for example.

Since ECL is quite slow anyways, I think these drawbacks are worth it
since the biggest reason for using ECL would be to get a statically
linked binary. If we change our minds, it shouldn't be too hard to
provide ecl-static and ecl-dynamic as separate implementations.

ECL is LGPL and some libraries it uses as part of its runtime are as
well. I've outlined in the ecl-static overlay why this should be of no
concern in the context of depot even though we are statically linking.

Currently everything is building except projects that are using cffi to
load shared libaries which have gotten an appropriate
`badImplementations` entry. To get the rest building the following
changes were made:

* Anywhere a dependency on UIOP is expressed as `bundled "uiop"` we now
  use `bundled "asdf"` for all implementations except SBCL. From my
  testing, SBCL seems to be the only implementation to support using
  `(require 'uiop)` to only load the UIOP package. Where both a
  dependency on ASDF and UIOP exists, we just delete the UIOP one.
  `(require 'asdf)` always causes UIOP to be available.

* Where appropriate only conditionally compile SBCL-specific code and
  if any build the corresponding files for ECL.

* //lisp/klatre: Use the standard condition parse-error for all
  implementations except SBCL in try-parse-integer.

* //3p/lisp/ironclad: disable SBCL assembly optimization hack for all
  other platforms as it may interfere with compilation.

* //3p/lisp/trivial-mimes: prevent call to asdf function by substituting
  it out of the source since it always errors out in ECL and we hardcode
  the correct path elsewhere anyways.

As it stands ECL still suffers from a very weird problem which happens
when compiling postmodern and moptilities:
https://gitlab.com/embeddable-common-lisp/ecl/-/issues/651

Change-Id: I0285924f92ac154126b4c42145073c3fb33702ed
Reviewed-on: https://cl.tvl.fyi/c/depot/+/3297
Tested-by: BuildkiteCI
Reviewed-by: tazjin <mail@tazj.in>
Reviewed-by: eta <tvl@eta.st>

1 files changed, 157 insertions, 4 deletions

diff --git a/nix/buildLisp/default.nix b/nix/buildLisp/default.nix
index c214a542de..ec42cc66f3 100644
--- a/nix/buildLisp/default.nix
+++ b/nix/buildLisp/default.nix
@@ -8,7 +8,7 @@
 
 let
   inherit (builtins) map elemAt match filter;
-  inherit (pkgs) lib runCommandNoCC makeWrapper writeText writeShellScriptBin sbcl;
+  inherit (pkgs) lib runCommandNoCC makeWrapper writeText writeShellScriptBin sbcl ecl-static;
 
   #
   # Internal helper definitions
@@ -16,6 +16,19 @@ let
 
   defaultImplementation = "sbcl";
 
+  # Many Common Lisp implementations (like ECL and CCL) will occasionally drop
+  # you into an interactive debugger even when executing something as a script.
+  # In nix builds we don't want such a situation: Any error should make the
+  # script exit non-zero. Luckily the ANSI standard specifies *debugger-hook*
+  # which is invoked before the debugger letting us just do that.
+  disableDebugger = writeText "disable-debugger.lisp" ''
+    (setf *debugger-hook*
+          (lambda (error hook)
+            (declare (ignore hook))
+            (format *error-output* "~%Unhandled error: ~a~%" error)
+            #+ecl (ext:quit 1)))
+  '';
+
   # Process a list of arbitrary values which also contains “implementation
   # filter sets” which describe conditonal inclusion of elements depending
   # on the CL implementation used. Elements are processed in the following
@@ -199,6 +212,9 @@ let
   #   executable which runs 'main' (and exits) where 'main' is available from
   #   'deps'. The executable should be created as "$out/bin/${name}", usually
   #   by dumping the lisp image with the replaced toplevel function replaced.
+  # - wrapProgram :: boolean
+  #   Whether to wrap the resulting binary / image with a wrapper script setting
+  #   `LD_LIBRARY_PATH`.
   # - genTestLisp :: { name, srcs, deps, expression } -> file
   #   Builds a lisp file which loads the given 'deps' and 'srcs' files and
   #   then evaluates 'expression'. Depending on whether 'expression' returns
@@ -291,6 +307,8 @@ let
                              :purify t))
       '';
 
+      wrapProgram = true;
+
       genTestLisp = genTestLispGeneric impls.sbcl;
 
       lispWith = deps:
@@ -304,6 +322,141 @@ let
           } $@
         '';
     };
+    ecl = {
+      runScript = "${ecl-static}/bin/ecl --load ${disableDebugger} --shell";
+      faslExt = "fasc";
+      genLoadLisp = genLoadLispGeneric impls.ecl;
+      genCompileLisp = { name, srcs, deps }: writeText "ecl-compile.lisp" ''
+        ;; This seems to be required to bring make the 'c' package available
+        ;; early, otherwise ECL tends to fail with a read failure…
+        (ext:install-c-compiler)
+
+        ;; Load dependencies
+        ${impls.ecl.genLoadLisp deps}
+
+        (defun getenv-or-fail (var)
+          (or (ext:getenv var)
+              (error (format nil "Missing expected environment variable ~A" var))))
+
+        (defun nix-compile-file (srcfile &key native)
+          "Compile the given srcfile into a compilation unit in :out-dir using
+          a unique name based on srcfile as the filename which is returned after
+          compilation. If :native is true, create an native object file,
+          otherwise a byte-compile fasc file is built and immediately loaded."
+
+          (let* ((unique-name (substitute #\_ #\/ srcfile))
+                 (out-file (make-pathname :type (if native "o" "fasc")
+                                          :directory (getenv-or-fail "NIX_BUILD_TOP")
+                                          :name unique-name)))
+            (multiple-value-bind (out-truename _warnings-p failure-p)
+                (compile-file srcfile :system-p native
+                                      :load (not native)
+                                      :output-file out-file
+                                      :verbose t :print t)
+              (if failure-p (ext:quit 1) out-truename))))
+
+        (let* ((out-dir (getenv-or-fail "out"))
+               (nix-build-dir (getenv-or-fail "NIX_BUILD_TOP"))
+               (srcs
+                ;; These forms are inserted by the Nix build
+                '(${lib.concatMapStringsSep "\n" (src: "\"${src}\"") srcs})))
+
+          ;; First, we'll byte compile loadable FASL files and load them
+          ;; immediately. Since we are using a statically linked ECL, there's
+          ;; no way to load native objects, so we rely on byte compilation
+          ;; for all our loading — which is crucial in compilation of course.
+          (ext:install-bytecodes-compiler)
+
+          ;; ECL's bytecode FASLs can just be concatenated to create a bundle
+          ;; at least since a recent bugfix which we apply as a patch.
+          ;; See also: https://gitlab.com/embeddable-common-lisp/ecl/-/issues/649
+          (let ((bundle-out (make-pathname :type "fasc" :name "${name}"
+                                           :directory out-dir)))
+
+            (with-open-file (fasc-stream bundle-out :direction :output)
+              (ext:run-program "cat"
+                               (mapcar (lambda (f)
+                                         (namestring
+                                          (nix-compile-file f :native nil)))
+                                       srcs)
+                               :output fasc-stream)))
+
+          (ext:install-c-compiler)
+
+          ;; Build a (natively compiled) static archive (.a) file. We want to
+          ;; use this for (statically) linking an executable later. The bytecode
+          ;; dance is only required because we can't load such archives.
+          (c:build-static-library
+           (make-pathname :type "a" :name "${name}" :directory out-dir)
+           :lisp-files (mapcar (lambda (x)
+                                 (nix-compile-file x :native t))
+                               srcs)))
+      '';
+      genDumpLisp = { name, main, deps }: writeText "ecl-dump.lisp" ''
+        (defun getenv-or-fail (var)
+          (or (ext:getenv var)
+              (error (format nil "Missing expected environment variable ~A" var))))
+
+        ${impls.ecl.genLoadLisp deps}
+
+        ;; makes a 'c' package available that can link executables
+        (ext:install-c-compiler)
+
+        (c:build-program
+         (make-pathname :name "${name}"
+                        :directory (concatenate 'string
+                                                (getenv-or-fail "out")
+                                                "/bin"))
+         :epilogue-code `(progn
+                          ;; UIOP doesn't understand ECL, so we need to make it
+                          ;; aware that we are a proper executable, causing it
+                          ;; to handle argument parsing and such properly. Since
+                          ;; this needs to work even when we're not using UIOP,
+                          ;; we need to do some compile-time acrobatics.
+                          ,(when (find-package 'uiop)
+                            `(setf ,(find-symbol "*IMAGE-DUMPED-P*" :uiop) :executable))
+                          ;; Run the actual application…
+                          (${main})
+                          ;; … and exit.
+                          (ext:quit))
+         ;; ECL can't remember these from its own build…
+         :ld-flags '("-static")
+         :lisp-files
+         ;; The following forms are inserted by the Nix build
+         '(${
+             lib.concatMapStrings (dep: ''
+               "${dep}/${dep.lispName}.a"
+             '') (allDeps impls.ecl deps)
+           }))
+      '';
+
+      wrapProgram = false;
+
+      genTestLisp = genTestLispGeneric impls.ecl;
+
+      lispWith = deps:
+        let lispDeps = filter (d: !d.lispBinary) (allDeps impls.ecl deps);
+        in writeShellScriptBin "ecl" ''
+          exec ${ecl-static}/bin/ecl ${
+            lib.optionalString (deps != [])
+              "--load ${writeText "load.lisp" (impls.ecl.genLoadLisp lispDeps)}"
+          } $@
+        '';
+
+      bundled = name: runCommandNoCC "${name}-cllib" {
+        passthru = {
+          lispName = name;
+          lispNativeDeps = [];
+          lispDeps = [];
+          lispBinary = false;
+          repl = impls.ecl.lispWith [ (impls.ecl.bundled name) ];
+        };
+      } ''
+        mkdir -p "$out"
+        ln -s "${ecl-static}/lib/ecl-${ecl-static.version}/${name}.${impls.ecl.faslExt}" -t "$out"
+        ln -s "${ecl-static}/lib/ecl-${ecl-static.version}/lib${name}.a" "$out/${name}.a"
+      '';
+    };
   };
 
   #
@@ -412,7 +565,7 @@ let
         lispBinary = true;
         tests = testDrv;
       };
-    } ''
+    } (''
       ${if ! isNull testDrv
         then "echo 'Test ${testDrv} succeeded'"
         else ""}
@@ -424,9 +577,9 @@ let
           deps = ([ selfLib ] ++ lispDeps);
         }
       }
-
+    '' + lib.optionalString impl.wrapProgram ''
       wrapProgram $out/bin/${name} --prefix LD_LIBRARY_PATH : "${libPath}"
-    '');
+    ''));
 
   # 'bundled' creates a "library" which makes a built-in package available,
   # such as any of SBCL's sb-* packages or ASDF. By default this is done