From d7e70b1d7270548272267e9b01dc9867f94f5f74 Mon Sep 17 00:00:00 2001 From: sterni Date: Fri, 13 Aug 2021 22:24:50 +0200 Subject: feat(nix/buildLisp): add ccl This adds support for Clozure's CL implementation to buildLisp. This is quite trivial in comparison to ECL since SBCL and CCL have very similar in how they work (so much so that CCL also suffers from b/136). Also the similarities in the code actually added here are striking, so I'll try to make an effort to reduce the code duplication in the future. To fix builds with CCL the following changes were made: * //3p/lisp/nibbles: The double inclusion of the types.lisp file was fixed. CCL doesn't like double definitions and refuses to compile otherwise. * //3p/lisp/physical-quantities: Update to a new bug fix release which contains a compilation fix for CCL. * //3p/lisp/routes: apply a patch fixing the build which was previously failing due to a double definition. * //3p/lisp/usocket: only depend on sb-bsd-sockets for SBCL and ECL, the latter of which seems to have a SBCL compatible implementation of the package. * Conditionally include a few CCL-specific source files and add `badImplementation` entries for the remaining failures which are //fun/gemma (to be expected) and //web/panettone which fails with an incredibly vague message. Change-Id: I666efdc39a0f16ee1bb6e23225784c709b04e740 Reviewed-on: https://cl.tvl.fyi/c/depot/+/3350 Tested-by: BuildkiteCI Reviewed-by: tazjin --- fun/gemma/default.nix | 1 + nix/buildLisp/default.nix | 104 ++++++++++++++++++++++++++++++- third_party/lisp/cffi.nix | 1 + third_party/lisp/cl-fad.nix | 3 + third_party/lisp/cl-plus-ssl.nix | 7 ++- third_party/lisp/closer-mop.nix | 1 + third_party/lisp/ironclad.nix | 2 + third_party/lisp/nibbles.nix | 1 - third_party/lisp/physical-quantities.nix | 5 +- third_party/lisp/routes.nix | 22 +++++-- third_party/lisp/trivial-features.nix | 1 + third_party/lisp/usocket.nix | 9 ++- web/panettone/default.nix | 1 + 13 files changed, 145 insertions(+), 13 deletions(-) diff --git a/fun/gemma/default.nix b/fun/gemma/default.nix index 20acace2d0..0b5e67b159 100644 --- a/fun/gemma/default.nix +++ b/fun/gemma/default.nix @@ -50,6 +50,7 @@ in depot.nix.buildLisp.program { # depends on SBCL brokenOn = [ + "ccl" "ecl" ]; } diff --git a/nix/buildLisp/default.nix b/nix/buildLisp/default.nix index ec42cc66f3..4e83cc2a6b 100644 --- a/nix/buildLisp/default.nix +++ b/nix/buildLisp/default.nix @@ -8,7 +8,8 @@ let inherit (builtins) map elemAt match filter; - inherit (pkgs) lib runCommandNoCC makeWrapper writeText writeShellScriptBin sbcl ecl-static; + inherit (pkgs) lib runCommandNoCC makeWrapper writeText writeShellScriptBin sbcl ecl-static ccl; + inherit (pkgs.stdenv) targetPlatform; # # Internal helper definitions @@ -26,6 +27,7 @@ let (lambda (error hook) (declare (ignore hook)) (format *error-output* "~%Unhandled error: ~a~%" error) + #+ccl (quit 1) #+ecl (ext:quit 1))) ''; @@ -457,6 +459,106 @@ let ln -s "${ecl-static}/lib/ecl-${ecl-static.version}/lib${name}.a" "$out/${name}.a" ''; }; + ccl = { + # Relatively bespoke wrapper script necessary to make CCL just™ execute + # a lisp file as a script. + runScript = pkgs.writers.writeBash "ccl" '' + # don't print intro message etc. + args=("--quiet") + + # makes CCL crash on error instead of entering the debugger + args+=("--load" "${disableDebugger}") + + # load files from command line in order + for f in "$@"; do + args+=("--load" "$f") + done + + # Exit if everything was processed successfully + args+=("--eval" "(quit)") + + exec ${ccl}/bin/ccl ''${args[@]} + ''; + + # See https://ccl.clozure.com/docs/ccl.html#building-definitions + faslExt = + /**/ if targetPlatform.isPowerPC && targetPlatform.is32bit then "pfsl" + else if targetPlatform.isPowerPC && targetPlatform.is64bit then "p64fsl" + else if targetPlatform.isx86_64 && targetPlatform.isLinux then "lx64fsl" + else if targetPlatform.isx86_32 && targetPlatform.isLinux then "lx32fsl" + else if targetPlatform.isAarch32 && targetPlatform.isLinux then "lafsl" + else if targetPlatform.isx86_32 && targetPlatform.isDarwin then "dx32fsl" + else if targetPlatform.isx86_64 && targetPlatform.isDarwin then "dx64fsl" + else if targetPlatform.isx86_64 && targetPlatform.isDarwin then "dx64fsl" + else if targetPlatform.isx86_32 && targetPlatform.isFreeBSD then "fx32fsl" + else if targetPlatform.isx86_64 && targetPlatform.isFreeBSD then "fx64fsl" + else if targetPlatform.isx86_32 && targetPlatform.isWindows then "wx32fsl" + else if targetPlatform.isx86_64 && targetPlatform.isWindows then "wx64fsl" + else builtins.throw "Don't know what FASLs are called for this platform: " + + pkgs.stdenv.targetPlatform.system; + + genLoadLisp = genLoadLispGeneric impls.ccl; + + genCompileLisp = { name, srcs, deps }: writeText "ccl-compile.lisp" '' + ${impls.ccl.genLoadLisp deps} + + (defun getenv-or-fail (var) + (or (getenv var) + (error (format nil "Missing expected environment variable ~A" var)))) + + (defun nix-compile-file (srcfile) + "Trivial wrapper around COMPILE-FILE which causes CCL to exit if + compilation fails and LOADs the compiled file on success." + (let ((output (make-pathname :name (substitute #\_ #\/ srcfile) + :type "${impls.ccl.faslExt}" + :directory (getenv-or-fail "NIX_BUILD_TOP")))) + (multiple-value-bind (out-truename _warnings-p failure-p) + (compile-file srcfile :output-file output :print t :verbose t) + (declare (ignore _warnings-p)) + (if failure-p (quit 1) + (progn (load out-truename) out-truename))))) + + (fasl-concatenate (make-pathname :name "${name}" :type "${impls.ccl.faslExt}" + :directory (getenv-or-fail "out")) + (mapcar #'nix-compile-file + ;; These forms where inserted by the Nix build + '(${ + lib.concatMapStrings (src: '' + "${src}" + '') srcs + }))) + ''; + + genDumpLisp = { name, main, deps }: writeText "ccl-dump.lisp" '' + ${impls.ccl.genLoadLisp deps} + + (let* ((out (or (getenv "out") (error "Not running in a Nix build"))) + (bindir (concatenate 'string out "/bin/")) + (executable (make-pathname :directory bindir :name "${name}"))) + + (save-application executable + :purify t + :error-handler :quit + :toplevel-function (function ${main}) + :mode #o755 + ;; TODO(sterni): use :native t on macOS + :prepend-kernel t)) + ''; + + wrapProgram = true; + + genTestLisp = genTestLispGeneric impls.ccl; + + lispWith = deps: + let lispDeps = filter (d: !d.lispBinary) (allDeps impls.ccl deps); + in writeShellScriptBin "ccl" '' + export LD_LIBRARY_PATH="${lib.makeLibraryPath (allNative [] lispDeps)}" + exec ${ccl}/bin/ccl ${ + lib.optionalString (deps != []) + "--load ${writeText "load.lisp" (impls.ccl.genLoadLisp lispDeps)}" + } "$@" + ''; + }; }; # diff --git a/third_party/lisp/cffi.nix b/third_party/lisp/cffi.nix index 89fe9fcad4..c8d240c8cd 100644 --- a/third_party/lisp/cffi.nix +++ b/third_party/lisp/cffi.nix @@ -19,6 +19,7 @@ in buildLisp.library { { ecl = src + "/src/cffi-ecl.lisp"; sbcl = src + "/src/cffi-sbcl.lisp"; + ccl = src + "/src/cffi-openmcl.lisp"; } ] ++ map (f: src + ("/src/" + f)) [ "package.lisp" diff --git a/third_party/lisp/cl-fad.nix b/third_party/lisp/cl-fad.nix index 2249db66ac..905e0821ca 100644 --- a/third_party/lisp/cl-fad.nix +++ b/third_party/lisp/cl-fad.nix @@ -22,6 +22,9 @@ in buildLisp.library { srcs = map (f: src + ("/" + f)) [ "packages.lisp" + ] ++ [ + { ccl = "${src}/openmcl.lisp"; } + ] ++ map (f: src + ("/" + f)) [ "fad.lisp" "path.lisp" "temporary-files.lisp" diff --git a/third_party/lisp/cl-plus-ssl.nix b/third_party/lisp/cl-plus-ssl.nix index 1dab7c3abb..e6a616b95e 100644 --- a/third_party/lisp/cl-plus-ssl.nix +++ b/third_party/lisp/cl-plus-ssl.nix @@ -17,8 +17,11 @@ in buildLisp.library { trivial-features trivial-garbage trivial-gray-streams - (buildLisp.bundled "uiop") - (buildLisp.bundled "sb-posix") + { + scbl = buildLisp.bundled "uiop"; + default = buildLisp.bundled "asdf"; + } + { sbcl = buildLisp.bundled "sb-posix"; } ]; native = [ pkgs.openssl ]; diff --git a/third_party/lisp/closer-mop.nix b/third_party/lisp/closer-mop.nix index d6a677625e..21fb3ba14b 100644 --- a/third_party/lisp/closer-mop.nix +++ b/third_party/lisp/closer-mop.nix @@ -18,6 +18,7 @@ in depot.nix.buildLisp.library { { sbcl = "${src}/closer-sbcl.lisp"; ecl = "${src}/closer-ecl.lisp"; + ccl = "${src}/closer-clozure.lisp"; } ]; } diff --git a/third_party/lisp/ironclad.nix b/third_party/lisp/ironclad.nix index fe0e052c32..cdd829924e 100644 --- a/third_party/lisp/ironclad.nix +++ b/third_party/lisp/ironclad.nix @@ -43,6 +43,8 @@ in depot.nix.buildLisp.library { { sbcl = getSrc "opt/sbcl/x86oid-vm.lisp"; } { ecl = getSrc "opt/ecl/c-functions.lisp"; } + + { ccl = getSrc "opt/ccl/x86oid-vm.lisp"; } ] ++ map getSrc [ "common.lisp" diff --git a/third_party/lisp/nibbles.nix b/third_party/lisp/nibbles.nix index b797c83a5f..da542fe911 100644 --- a/third_party/lisp/nibbles.nix +++ b/third_party/lisp/nibbles.nix @@ -21,7 +21,6 @@ in depot.nix.buildLisp.library { "package.lisp" "types.lisp" "macro-utils.lisp" - "types.lisp" "vectors.lisp" "streams.lisp" ] ++ [ diff --git a/third_party/lisp/physical-quantities.nix b/third_party/lisp/physical-quantities.nix index 6e124a9132..b8079e8eba 100644 --- a/third_party/lisp/physical-quantities.nix +++ b/third_party/lisp/physical-quantities.nix @@ -4,9 +4,10 @@ let src = pkgs.fetchFromGitHub { owner = "mrossini-ethz"; repo = "physical-quantities"; - rev = "8feb66ef3293fcb9ff4c4bd3ee872bfc385a590e"; - sha256 = "1qznv0hmn2n7g9dxx1iw0qpr0pf2lnbahn0x0b3v50xzcb65kgig"; + rev = "v0.2.1"; + sha256 = "0mb2s94s6fhw5vfa89naalw7ld11sdsszlqpz0c65dvpfyfmmdmh"; }; + in depot.nix.buildLisp.library { name = "physical-quantities"; diff --git a/third_party/lisp/routes.nix b/third_party/lisp/routes.nix index 61c6749660..a76912c651 100644 --- a/third_party/lisp/routes.nix +++ b/third_party/lisp/routes.nix @@ -2,11 +2,22 @@ let - src = pkgs.fetchFromGitHub { - owner = "archimag"; - repo = "cl-routes"; - rev = "1b79e85aa653e1ec87e21ca745abe51547866fa9"; - sha256 = "1zpk3cp2v8hm50ppjl10yxr437vv4552r8hylvizglzrq2ibsbr1"; + src = pkgs.applyPatches { + name = "routes-source"; + src = pkgs.fetchFromGitHub { + owner = "archimag"; + repo = "cl-routes"; + rev = "1b79e85aa653e1ec87e21ca745abe51547866fa9"; + sha256 = "1zpk3cp2v8hm50ppjl10yxr437vv4552r8hylvizglzrq2ibsbr1"; + }; + + patches = [ + (pkgs.fetchpatch { + name = "fix-build-with-ccl.patch"; + url = "https://github.com/archimag/cl-routes/commit/2296cdc316ef8e34310f2718b5d35a30040deee0.patch"; + sha256 = "007c19kmymalam3v6l6y2qzch8xs3xnphrcclk1jrpggvigcmhax"; + }) + ]; }; in depot.nix.buildLisp.library { @@ -24,5 +35,4 @@ in depot.nix.buildLisp.library { "route.lisp" "mapper.lisp" ]; - } diff --git a/third_party/lisp/trivial-features.nix b/third_party/lisp/trivial-features.nix index 3ad424b8ab..13a63bab1a 100644 --- a/third_party/lisp/trivial-features.nix +++ b/third_party/lisp/trivial-features.nix @@ -10,6 +10,7 @@ in depot.nix.buildLisp.library { { sbcl = src + "/src/tf-sbcl.lisp"; ecl = src + "/src/tf-ecl.lisp"; + ccl = src + "/src/tf-openmcl.lisp"; } ]; } diff --git a/third_party/lisp/usocket.nix b/third_party/lisp/usocket.nix index dbbfd2fbf1..dc4281c795 100644 --- a/third_party/lisp/usocket.nix +++ b/third_party/lisp/usocket.nix @@ -14,7 +14,10 @@ in buildLisp.library { name = "usocket"; deps = with depot.third_party.lisp; [ (buildLisp.bundled "asdf") - (buildLisp.bundled "sb-bsd-sockets") + { + ecl = buildLisp.bundled "sb-bsd-sockets"; + sbcl = buildLisp.bundled "sb-bsd-sockets"; + } split-sequence ]; @@ -40,5 +43,9 @@ in buildLisp.library { # but usocket also has some ECL specific code { ecl = "${src}/backend/sbcl.lisp"; } { ecl = "${src}/backend/ecl.lisp"; } + + # Same for CCL + { ccl = "${src}/backend/openmcl.lisp"; } + { ccl = "${src}/backend/clozure.lisp"; } ]); } diff --git a/web/panettone/default.nix b/web/panettone/default.nix index 8dd4543af0..a01e0d81c8 100644 --- a/web/panettone/default.nix +++ b/web/panettone/default.nix @@ -50,5 +50,6 @@ depot.nix.buildLisp.program { brokenOn = [ "ecl" # dependencies use dynamic cffi + "ccl" # The value NIL is not of the expected type STRING. when loading model.lisp ]; } -- cgit 1.4.1