feat(nix/buildLisp): add ccl r/2772

This adds support for Clozure's CL implementation to buildLisp. This is
quite trivial in comparison to ECL since SBCL and CCL have very similar
in how they work (so much so that CCL also suffers from b/136).

Also the similarities in the code actually added here are striking, so
I'll try to make an effort to reduce the code duplication in the
future.

To fix builds with CCL the following changes were made:

* //3p/lisp/nibbles: The double inclusion of the types.lisp file was
  fixed. CCL doesn't like double definitions and refuses to compile
  otherwise.

* //3p/lisp/physical-quantities: Update to a new bug fix release which
  contains a compilation fix for CCL.

* //3p/lisp/routes: apply a patch fixing the build which was previously
  failing due to a double definition.

* //3p/lisp/usocket: only depend on sb-bsd-sockets for SBCL and ECL, the
  latter of which seems to have a SBCL compatible implementation of the
  package.

* Conditionally include a few CCL-specific source files and add
  `badImplementation` entries for the remaining failures which are
  //fun/gemma (to be expected) and //web/panettone which fails with an
  incredibly vague message.

Change-Id: I666efdc39a0f16ee1bb6e23225784c709b04e740
Reviewed-on: https://cl.tvl.fyi/c/depot/+/3350
Tested-by: BuildkiteCI
Reviewed-by: tazjin <mail@tazj.in>

13 files changed, 145 insertions, 13 deletions

diff --git a/fun/gemma/default.nix b/fun/gemma/default.nix
index 20acace2d007..0b5e67b1594b 100644
--- a/fun/gemma/default.nix
+++ b/fun/gemma/default.nix
@@ -50,6 +50,7 @@ in depot.nix.buildLisp.program {
 
   # depends on SBCL
   brokenOn = [
+    "ccl"
     "ecl"
   ];
 }
diff --git a/nix/buildLisp/default.nix b/nix/buildLisp/default.nix
index ec42cc66f30b..4e83cc2a6b63 100644
--- a/nix/buildLisp/default.nix
+++ b/nix/buildLisp/default.nix
@@ -8,7 +8,8 @@
 
 let
   inherit (builtins) map elemAt match filter;
-  inherit (pkgs) lib runCommandNoCC makeWrapper writeText writeShellScriptBin sbcl ecl-static;
+  inherit (pkgs) lib runCommandNoCC makeWrapper writeText writeShellScriptBin sbcl ecl-static ccl;
+  inherit (pkgs.stdenv) targetPlatform;
 
   #
   # Internal helper definitions
@@ -26,6 +27,7 @@ let
           (lambda (error hook)
             (declare (ignore hook))
             (format *error-output* "~%Unhandled error: ~a~%" error)
+            #+ccl (quit 1)
             #+ecl (ext:quit 1)))
   '';
 
@@ -457,6 +459,106 @@ let
         ln -s "${ecl-static}/lib/ecl-${ecl-static.version}/lib${name}.a" "$out/${name}.a"
       '';
     };
+    ccl = {
+      # Relatively bespoke wrapper script necessary to make CCL just™ execute
+      # a lisp file as a script.
+      runScript = pkgs.writers.writeBash "ccl" ''
+        # don't print intro message etc.
+        args=("--quiet")
+
+        # makes CCL crash on error instead of entering the debugger
+        args+=("--load" "${disableDebugger}")
+
+        # load files from command line in order
+        for f in "$@"; do
+          args+=("--load" "$f")
+        done
+
+        # Exit if everything was processed successfully
+        args+=("--eval" "(quit)")
+
+        exec ${ccl}/bin/ccl ''${args[@]}
+      '';
+
+      # See https://ccl.clozure.com/docs/ccl.html#building-definitions
+      faslExt =
+        /**/ if targetPlatform.isPowerPC && targetPlatform.is32bit then "pfsl"
+        else if targetPlatform.isPowerPC && targetPlatform.is64bit then "p64fsl"
+        else if targetPlatform.isx86_64 && targetPlatform.isLinux then "lx64fsl"
+        else if targetPlatform.isx86_32 && targetPlatform.isLinux then "lx32fsl"
+        else if targetPlatform.isAarch32 && targetPlatform.isLinux then "lafsl"
+        else if targetPlatform.isx86_32 && targetPlatform.isDarwin then "dx32fsl"
+        else if targetPlatform.isx86_64 && targetPlatform.isDarwin then "dx64fsl"
+        else if targetPlatform.isx86_64 && targetPlatform.isDarwin then "dx64fsl"
+        else if targetPlatform.isx86_32 && targetPlatform.isFreeBSD then "fx32fsl"
+        else if targetPlatform.isx86_64 && targetPlatform.isFreeBSD then "fx64fsl"
+        else if targetPlatform.isx86_32 && targetPlatform.isWindows then "wx32fsl"
+        else if targetPlatform.isx86_64 && targetPlatform.isWindows then "wx64fsl"
+        else builtins.throw "Don't know what FASLs are called for this platform: "
+          + pkgs.stdenv.targetPlatform.system;
+
+      genLoadLisp = genLoadLispGeneric impls.ccl;
+
+      genCompileLisp = { name, srcs, deps }: writeText "ccl-compile.lisp" ''
+        ${impls.ccl.genLoadLisp deps}
+
+        (defun getenv-or-fail (var)
+          (or (getenv var)
+              (error (format nil "Missing expected environment variable ~A" var))))
+
+        (defun nix-compile-file (srcfile)
+          "Trivial wrapper around COMPILE-FILE which causes CCL to exit if
+          compilation fails and LOADs the compiled file on success."
+          (let ((output (make-pathname :name (substitute #\_ #\/ srcfile)
+                                       :type "${impls.ccl.faslExt}"
+                                       :directory (getenv-or-fail "NIX_BUILD_TOP"))))
+            (multiple-value-bind (out-truename _warnings-p failure-p)
+                (compile-file srcfile :output-file output :print t :verbose t)
+                (declare (ignore _warnings-p))
+              (if failure-p (quit 1)
+                  (progn (load out-truename) out-truename)))))
+
+        (fasl-concatenate (make-pathname :name "${name}" :type "${impls.ccl.faslExt}"
+                                         :directory (getenv-or-fail "out"))
+                          (mapcar #'nix-compile-file
+                                  ;; These forms where inserted by the Nix build
+                                  '(${
+                                      lib.concatMapStrings (src: ''
+                                        "${src}"
+                                      '') srcs
+                                   })))
+      '';
+
+      genDumpLisp = { name, main, deps }: writeText "ccl-dump.lisp" ''
+        ${impls.ccl.genLoadLisp deps}
+
+        (let* ((out (or (getenv "out") (error "Not running in a Nix build")))
+               (bindir (concatenate 'string out "/bin/"))
+               (executable (make-pathname :directory bindir :name "${name}")))
+
+          (save-application executable
+                            :purify t
+                            :error-handler :quit
+                            :toplevel-function (function ${main})
+                            :mode #o755
+                            ;; TODO(sterni): use :native t on macOS
+                            :prepend-kernel t))
+      '';
+
+      wrapProgram = true;
+
+      genTestLisp = genTestLispGeneric impls.ccl;
+
+      lispWith = deps:
+        let lispDeps = filter (d: !d.lispBinary) (allDeps impls.ccl deps);
+        in writeShellScriptBin "ccl" ''
+          export LD_LIBRARY_PATH="${lib.makeLibraryPath (allNative [] lispDeps)}"
+          exec ${ccl}/bin/ccl ${
+            lib.optionalString (deps != [])
+              "--load ${writeText "load.lisp" (impls.ccl.genLoadLisp lispDeps)}"
+          } "$@"
+      '';
+    };
   };
 
   #
diff --git a/third_party/lisp/cffi.nix b/third_party/lisp/cffi.nix
index 89fe9fcad4f4..c8d240c8cd0e 100644
--- a/third_party/lisp/cffi.nix
+++ b/third_party/lisp/cffi.nix
@@ -19,6 +19,7 @@ in buildLisp.library {
     {
       ecl = src + "/src/cffi-ecl.lisp";
       sbcl = src + "/src/cffi-sbcl.lisp";
+      ccl = src + "/src/cffi-openmcl.lisp";
     }
   ] ++ map (f: src + ("/src/" + f)) [
     "package.lisp"
diff --git a/third_party/lisp/cl-fad.nix b/third_party/lisp/cl-fad.nix
index 2249db66ac18..905e0821cac7 100644
--- a/third_party/lisp/cl-fad.nix
+++ b/third_party/lisp/cl-fad.nix
@@ -22,6 +22,9 @@ in buildLisp.library {
 
   srcs = map (f: src + ("/" + f)) [
     "packages.lisp"
+  ] ++ [
+    { ccl =  "${src}/openmcl.lisp"; }
+  ] ++ map (f: src + ("/" + f)) [
     "fad.lisp"
     "path.lisp"
     "temporary-files.lisp"
diff --git a/third_party/lisp/cl-plus-ssl.nix b/third_party/lisp/cl-plus-ssl.nix
index 1dab7c3abba4..e6a616b95e46 100644
--- a/third_party/lisp/cl-plus-ssl.nix
+++ b/third_party/lisp/cl-plus-ssl.nix
@@ -17,8 +17,11 @@ in buildLisp.library {
     trivial-features
     trivial-garbage
     trivial-gray-streams
-    (buildLisp.bundled "uiop")
-    (buildLisp.bundled "sb-posix")
+    {
+      scbl = buildLisp.bundled "uiop";
+      default = buildLisp.bundled "asdf";
+    }
+    { sbcl = buildLisp.bundled "sb-posix"; }
   ];
 
   native = [ pkgs.openssl ];
diff --git a/third_party/lisp/closer-mop.nix b/third_party/lisp/closer-mop.nix
index d6a677625e4d..21fb3ba14bef 100644
--- a/third_party/lisp/closer-mop.nix
+++ b/third_party/lisp/closer-mop.nix
@@ -18,6 +18,7 @@ in depot.nix.buildLisp.library {
     {
       sbcl = "${src}/closer-sbcl.lisp";
       ecl = "${src}/closer-ecl.lisp";
+      ccl = "${src}/closer-clozure.lisp";
     }
   ];
 }
diff --git a/third_party/lisp/ironclad.nix b/third_party/lisp/ironclad.nix
index fe0e052c32bc..cdd829924ee5 100644
--- a/third_party/lisp/ironclad.nix
+++ b/third_party/lisp/ironclad.nix
@@ -43,6 +43,8 @@ in depot.nix.buildLisp.library {
     { sbcl = getSrc "opt/sbcl/x86oid-vm.lisp"; }
 
     { ecl = getSrc "opt/ecl/c-functions.lisp"; }
+
+    { ccl = getSrc "opt/ccl/x86oid-vm.lisp"; }
   ] ++ map getSrc [
 
     "common.lisp"
diff --git a/third_party/lisp/nibbles.nix b/third_party/lisp/nibbles.nix
index b797c83a5fa1..da542fe91130 100644
--- a/third_party/lisp/nibbles.nix
+++ b/third_party/lisp/nibbles.nix
@@ -21,7 +21,6 @@ in depot.nix.buildLisp.library {
     "package.lisp"
     "types.lisp"
     "macro-utils.lisp"
-    "types.lisp"
     "vectors.lisp"
     "streams.lisp"
   ] ++ [
diff --git a/third_party/lisp/physical-quantities.nix b/third_party/lisp/physical-quantities.nix
index 6e124a913209..b8079e8ebaaf 100644
--- a/third_party/lisp/physical-quantities.nix
+++ b/third_party/lisp/physical-quantities.nix
@@ -4,9 +4,10 @@ let
   src = pkgs.fetchFromGitHub {
     owner = "mrossini-ethz";
     repo = "physical-quantities";
-    rev = "8feb66ef3293fcb9ff4c4bd3ee872bfc385a590e";
-    sha256 = "1qznv0hmn2n7g9dxx1iw0qpr0pf2lnbahn0x0b3v50xzcb65kgig";
+    rev = "v0.2.1";
+    sha256 = "0mb2s94s6fhw5vfa89naalw7ld11sdsszlqpz0c65dvpfyfmmdmh";
   };
+
 in depot.nix.buildLisp.library {
   name = "physical-quantities";
 
diff --git a/third_party/lisp/routes.nix b/third_party/lisp/routes.nix
index 61c674966094..a76912c651a9 100644
--- a/third_party/lisp/routes.nix
+++ b/third_party/lisp/routes.nix
@@ -2,11 +2,22 @@
 
 let
 
-  src = pkgs.fetchFromGitHub {
-    owner = "archimag";
-    repo = "cl-routes";
-    rev = "1b79e85aa653e1ec87e21ca745abe51547866fa9";
-    sha256 = "1zpk3cp2v8hm50ppjl10yxr437vv4552r8hylvizglzrq2ibsbr1";
+  src = pkgs.applyPatches {
+    name = "routes-source";
+    src = pkgs.fetchFromGitHub {
+      owner = "archimag";
+      repo = "cl-routes";
+      rev = "1b79e85aa653e1ec87e21ca745abe51547866fa9";
+      sha256 = "1zpk3cp2v8hm50ppjl10yxr437vv4552r8hylvizglzrq2ibsbr1";
+    };
+
+    patches = [
+      (pkgs.fetchpatch {
+        name = "fix-build-with-ccl.patch";
+        url = "https://github.com/archimag/cl-routes/commit/2296cdc316ef8e34310f2718b5d35a30040deee0.patch";
+        sha256 = "007c19kmymalam3v6l6y2qzch8xs3xnphrcclk1jrpggvigcmhax";
+      })
+    ];
   };
 
 in depot.nix.buildLisp.library {
@@ -24,5 +35,4 @@ in depot.nix.buildLisp.library {
     "route.lisp"
     "mapper.lisp"
   ];
-
 }
diff --git a/third_party/lisp/trivial-features.nix b/third_party/lisp/trivial-features.nix
index 3ad424b8abdc..13a63bab1aba 100644
--- a/third_party/lisp/trivial-features.nix
+++ b/third_party/lisp/trivial-features.nix
@@ -10,6 +10,7 @@ in depot.nix.buildLisp.library {
     {
       sbcl = src + "/src/tf-sbcl.lisp";
       ecl = src + "/src/tf-ecl.lisp";
+      ccl = src + "/src/tf-openmcl.lisp";
     }
   ];
 }
diff --git a/third_party/lisp/usocket.nix b/third_party/lisp/usocket.nix
index dbbfd2fbf110..dc4281c795c0 100644
--- a/third_party/lisp/usocket.nix
+++ b/third_party/lisp/usocket.nix
@@ -14,7 +14,10 @@ in buildLisp.library {
   name = "usocket";
   deps = with depot.third_party.lisp; [
     (buildLisp.bundled "asdf")
-    (buildLisp.bundled "sb-bsd-sockets")
+    {
+      ecl = buildLisp.bundled "sb-bsd-sockets";
+      sbcl = buildLisp.bundled "sb-bsd-sockets";
+    }
     split-sequence
   ];
 
@@ -40,5 +43,9 @@ in buildLisp.library {
     # but usocket also has some ECL specific code
     { ecl = "${src}/backend/sbcl.lisp"; }
     { ecl = "${src}/backend/ecl.lisp"; }
+
+    # Same for CCL
+    { ccl = "${src}/backend/openmcl.lisp"; }
+    { ccl = "${src}/backend/clozure.lisp"; }
   ]);
 }
diff --git a/web/panettone/default.nix b/web/panettone/default.nix
index 8dd4543af0d2..a01e0d81c83a 100644
--- a/web/panettone/default.nix
+++ b/web/panettone/default.nix
@@ -50,5 +50,6 @@ depot.nix.buildLisp.program {
 
   brokenOn = [
     "ecl" # dependencies use dynamic cffi
+    "ccl" # The value NIL is not of the expected type STRING. when loading model.lisp
   ];
 }