path: root/nix/buildLisp/default.nix

# buildLisp provides Nix functions to build Common Lisp packages,
# targeting SBCL.
#
# buildLisp is designed to enforce conventions and do away with the
# free-for-all of existing Lisp build systems.

{ pkgs ? import <nixpkgs> {}, ... }:

let
  inherit (builtins) map elemAt match filter;
  inherit (pkgs) lib runCommandNoCC makeWrapper writeText writeShellScriptBin sbcl;

  #
  # Internal helper definitions
  #

  defaultImplementation = "sbcl";

  # Process a list of arbitrary values which also contains “implementation
  # filter sets” which describe conditonal inclusion of elements depending
  # on the CL implementation used. Elements are processed in the following
  # manner:
  #
  # * Paths, strings, derivations are left as is
  # * A non-derivation attribute set is processed like this:
  #   1. If it has an attribute equal to impl.name, replace with its value.
  #   2. Alternatively use the value of the "default" attribute.
  #   3. In all other cases delete the element from the list.
  #
  # This can be used to express dependencies or source files which are specific
  # to certain implementations:
  #
  #  srcs = [
  #    # mixable with unconditional entries
  #    ./package.lisp
  #
  #    # implementation specific source files
  #    {
  #      ccl = ./impl-ccl.lisp;
  #      sbcl = ./impl-sbcl.lisp;
  #      ecl = ./impl-ecl.lisp;
  #    }
  #  ];
  #
  #  deps = [
  #    # this dependency is ignored if impl.name != "sbcl"
  #    { sbcl = buildLisp.bundled "sb-posix"; }
  #
  #    # only special casing for a single implementation
  #    {
  #      sbcl = buildLisp.bundled "uiop";
  #      default = buildLisp.bundled "asdf";
  #    }
  #  ];
  implFilter = impl: xs:
    let
      isFilterSet = x: builtins.isAttrs x && !(lib.isDerivation x);
    in builtins.map (
      x: if isFilterSet x then x.${impl.name} or x.default else x
    ) (builtins.filter (
      x: !(isFilterSet x) || x ? ${impl.name} || x ? default
    ) xs);

  # Generates lisp code which instructs the given lisp implementation to load
  # all the given dependencies.
  genLoadLispGeneric = impl: deps:
    lib.concatStringsSep "\n"
      (map (lib: "(load \"${lib}/${lib.lispName}.${impl.faslExt}\")")
        (allDeps impl deps));

  # 'genTestLispGeneric' generates a Lisp file that loads all sources and deps
  # and executes expression for a given implementation description.
  genTestLispGeneric = impl: { name, srcs, deps, expression }: writeText "${name}.lisp" ''
    ;; Dependencies
    ${impl.genLoadLisp deps}

    ;; Sources
    ${lib.concatStringsSep "\n" (map (src: "(load \"${src}\")") srcs)}

    ;; Test expression
    (unless ${expression}
      (exit :code 1))
  '';

  # 'dependsOn' determines whether Lisp library 'b' depends on 'a'.
  dependsOn = a: b: builtins.elem a b.lispDeps;

  # 'allDeps' flattens the list of dependencies (and their
  # dependencies) into one ordered list of unique deps which
  # all use the given implementation.
  allDeps = impl: deps: let
    # The override _should_ propagate itself recursively, as every derivation
    # would only expose its actually used dependencies. Use implementation
    # attribute created by withExtras if present, override in all other cases
    # (mainly bundled).
    deps' = builtins.map (dep: dep."${impl.name}" or (dep.overrideLisp (_: {
      implementation = impl.name;
    }))) deps;
  in (lib.toposort dependsOn (lib.unique (
    lib.flatten (deps' ++ (map (d: d.lispDeps) deps'))
  ))).result;

  # 'allNative' extracts all native dependencies of a dependency list
  # to ensure that library load paths are set correctly during all
  # compilations and program assembly.
  allNative = native: deps: lib.unique (
    lib.flatten (native ++ (map (d: d.lispNativeDeps) deps))
  );

  # Add an `overrideLisp` attribute to a function result that works
  # similar to `overrideAttrs`, but is used specifically for the
  # arguments passed to Lisp builders.
  makeOverridable = f: orig: (f orig) // {
    overrideLisp = new: makeOverridable f (orig // (new orig));
  };

  # This is a wrapper arround 'makeOverridable' which performs its
  # function, but also adds a the following additional attributes to the
  # resulting derivation, namely a repl attribute which builds a `lispWith`
  # derivation for the current implementation and additional attributes for
  # every all implementations. So `drv.sbcl` would build the derivation
  # with SBCL regardless of what was specified in the initial arguments.
  withExtras = f: args:
    let
      drv = (makeOverridable f) args;
    in lib.fix (self:
      drv.overrideLisp (old:
        let
          implementation = old.implementation or defaultImplementation;
          brokenOn = old.brokenOn or [];
          targets = lib.subtractLists brokenOn
            (builtins.attrNames impls);
        in {
          passthru = (old.passthru or {}) // {
            repl = impls."${implementation}".lispWith [ self ];

            # meta is done via passthru to minimize rebuilds caused by overriding
            meta = (old.passthru.meta or {}) // {
              inherit targets;
            };
          } // builtins.listToAttrs (builtins.map (name: {
            inherit name;
            value = self.overrideLisp (_: {
              implementation = name;
            });
          }) (builtins.attrNames impls));
        }) // {
          overrideLisp = new: withExtras f (args // new args);
        });

  # 'testSuite' builds a Common Lisp test suite that loads all of srcs and deps,
  # and then executes expression to check its result
  testSuite = { name, expression, srcs, deps ? [], native ? [], impl }:
    let
      lispNativeDeps = allNative native deps;
      lispDeps = allDeps impl (implFilter impl deps);
      filteredSrcs = implFilter impl srcs;
    in runCommandNoCC name {
      LD_LIBRARY_PATH = lib.makeLibraryPath lispNativeDeps;
      LANG = "C.UTF-8";
    } ''
      echo "Running test suite ${name}"

      ${impl.runScript} ${
        impl.genTestLisp {
          inherit name expression;
          srcs = filteredSrcs;
          deps = lispDeps;
        }
      } | tee $out

      echo "Test suite ${name} succeeded"
    '';

  # 'impls' is an attribute set of attribute sets which describe how to do common
  # tasks when building for different Common Lisp implementations. Each
  # implementation set has the following members:
  #
  # Required members:
  #
  # - runScript :: string
  #   Describes how to invoke the implementation from the shell, so it runs a
  #   lisp file as a script and exits.
  # - faslExt :: string
  #   File extension of the implementations loadable (FASL) files.
  #   Implementations are free to generate native object files, but with the way
  #   buildLisp works it is required that we can also 'load' libraries, so
  #   (additionally) building a FASL or equivalent is required.
  # - genLoadLisp :: [ dependency ] -> string
  #   Returns lisp code to 'load' the given dependencies. 'genLoadLispGeneric'
  #   should work for most dependencies.
  # - genCompileLisp :: { name, srcs, deps } -> file
  #   Builds a lisp file which instructs the implementation to build a library
  #   from the given source files when executed. After running at least
  #   the file "$out/${name}.${impls.${implementation}.faslExt}" should have
  #   been created.
  # - genDumpLisp :: { name, main, deps } -> file
  #   Builds a lisp file which instructs the implementation to build an
  #   executable which runs 'main' (and exits) where 'main' is available from
  #   'deps'. The executable should be created as "$out/bin/${name}", usually
  #   by dumping the lisp image with the replaced toplevel function replaced.
  # - genTestLisp :: { name, srcs, deps, expression } -> file
  #   Builds a lisp file which loads the given 'deps' and 'srcs' files and
  #   then evaluates 'expression'. Depending on whether 'expression' returns
  #   true or false, the script must exit with a zero or non-zero exit code.
  #   'genTestLispGeneric' will work for most implementations.
  # - lispWith :: [ dependency ] -> drv
  #   Builds a script (or dumped image) which when executed loads (or has
  #   loaded) all given dependencies. When built this should create an executable
  #   at "$out/bin/${implementation}".
  #
  # Optional members:
  #
  # - bundled :: string -> library
  #   Allows giving an implementation specific builder for a bundled library.
  #   This function is used as a replacement for the internal defaultBundled
  #   function and only needs to support one implementation. The returned derivation
  #   must behave like one built by 'library' (in particular have the same files
  #   available in "$out" and the same 'passthru' attributes), but may be built
  #   completely differently.
  impls = lib.mapAttrs (name: v: { inherit name; } // v) {
    sbcl = {
      runScript = "${sbcl}/bin/sbcl --script";
      faslExt = "fasl";

      # 'genLoadLisp' generates Lisp code that instructs SBCL to load all
      # the provided Lisp libraries.
      genLoadLisp = genLoadLispGeneric impls.sbcl;

      # 'genCompileLisp' generates a Lisp file that instructs SBCL to
      # compile the provided list of Lisp source files to "$out/${name}.fasl".
      genCompileLisp = { name, srcs, deps }: writeText "sbcl-compile.lisp" ''
        ;; This file compiles the specified sources into the Nix build
        ;; directory, creating one FASL file for each source.
        (require 'sb-posix)

        ${impls.sbcl.genLoadLisp deps}

        (defun nix-compile-lisp (srcfile)
          (let ((outfile (make-pathname :type "fasl"
                                        :directory (or (sb-posix:getenv "NIX_BUILD_TOP")
                                                       (error "not running in a Nix build"))
                                        :name (substitute #\- #\/ srcfile))))
            (multiple-value-bind (out-truename _warnings-p failure-p)
                (compile-file srcfile :output-file outfile)
              (if failure-p (sb-posix:exit 1)
                  (progn
                    ;; For the case of multiple files belonging to the same
                    ;; library being compiled, load them in order:
                    (load out-truename)

                    ;; Return pathname as a string for cat-ting it later
                    (namestring out-truename))))))

        (let ((*compile-verbose* t)
              (catted-fasl (make-pathname :type "fasl"
                                          :directory (or (sb-posix:getenv "out")
                                                         (error "not running in a Nix build"))
                                          :name "${name}")))

          (with-open-file (file catted-fasl
                                :direction :output
                                :if-does-not-exist :create)

            ;; SBCL's FASL files can just be bundled together using cat
            (sb-ext:run-program "cat"
             (mapcar #'nix-compile-lisp
              ;; These forms were inserted by the Nix build:
              '(${
                lib.concatMapStringsSep "\n" (src: "\"${src}\"") srcs
              }))
             :output file :search t)))
      '';

      # 'genDumpLisp' generates a Lisp file that instructs SBCL to dump
      # the currently loaded image as an executable to $out/bin/$name.
      #
      # TODO(tazjin): Compression is currently unsupported because the
      # SBCL in nixpkgs is, by default, not compiled with zlib support.
      genDumpLisp = { name, main, deps }: writeText "sbcl-dump.lisp" ''
        (require 'sb-posix)

        ${impls.sbcl.genLoadLisp deps}

        (let* ((bindir (concatenate 'string (sb-posix:getenv "out") "/bin"))
               (outpath (make-pathname :name "${name}"
                                       :directory bindir)))
          (save-lisp-and-die outpath
                             :executable t
                             :toplevel (function ${main})
                             :purify t))
      '';

      genTestLisp = genTestLispGeneric impls.sbcl;

      lispWith = deps:
        let lispDeps = filter (d: !d.lispBinary) (allDeps impls.sbcl deps);
        in writeShellScriptBin "sbcl" ''
          export LD_LIBRARY_PATH="${lib.makeLibraryPath (allNative [] lispDeps)}"
          export LANG="C.UTF-8"
          exec ${sbcl}/bin/sbcl ${
            lib.optionalString (deps != [])
              "--load ${writeText "load.lisp" (impls.sbcl.genLoadLisp lispDeps)}"
          } $@
        '';
    };
  };

  #
  # Public API functions
  #

  # 'library' builds a list of Common Lisp files into an implementation
  # specific library format, usually a single FASL file, which can then be
  # loaded and built into an executable via 'program'.
  library =
    { name
    , implementation ? defaultImplementation
    , brokenOn ? [] # TODO(sterni): make this a warning
    , srcs
    , deps ? []
    , native ? []
    , tests ? null
    , passthru ? {}
    }:
    let
      impl = impls."${implementation}" or
        (builtins.throw "Unkown Common Lisp Implementation ${implementation}");
      filteredDeps = implFilter impl deps;
      filteredSrcs = implFilter impl srcs;
      lispNativeDeps = (allNative native filteredDeps);
      lispDeps = allDeps impl filteredDeps;
      testDrv = if ! isNull tests
        then testSuite {
          name = tests.name or "${name}-test";
          srcs = filteredSrcs ++ (tests.srcs or []);
          deps = filteredDeps ++ (tests.deps or []);
          expression = tests.expression;
          inherit impl;
        }
        else null;
    in lib.fix (self: runCommandNoCC "${name}-cllib" {
      LD_LIBRARY_PATH = lib.makeLibraryPath lispNativeDeps;
      LANG = "C.UTF-8";
      passthru = passthru // {
        inherit lispNativeDeps lispDeps;
        lispName = name;
        lispBinary = false;
        tests = testDrv;
      };
    } ''
      ${if ! isNull testDrv
        then "echo 'Test ${testDrv} succeeded'"
        else "echo 'No tests run'"}

      mkdir $out

      ${impl.runScript} ${
        impl.genCompileLisp {
          srcs = filteredSrcs;
          inherit name;
          deps = lispDeps;
        }
      }
    '');

  # 'program' creates an executable, usually containing a dumped image of the
  # specified sources and dependencies.
  program =
    { name
    , implementation ? defaultImplementation
    , brokenOn ? [] # TODO(sterni): make this a warning
    , main ? "${name}:main"
    , srcs
    , deps ? []
    , native ? []
    , tests ? null
    , passthru ? {}
    }:
    let
      impl = impls."${implementation}" or
        (builtins.throw "Unkown Common Lisp Implementation ${implementation}");
      filteredSrcs = implFilter impl srcs;
      filteredDeps = implFilter impl deps;
      lispDeps = allDeps impl filteredDeps;
      libPath = lib.makeLibraryPath (allNative native lispDeps);
      # overriding is used internally to propagate the implementation to use
      selfLib = (makeOverridable library) {
        inherit name native brokenOn;
        deps = lispDeps;
        srcs = filteredSrcs;
      };
      testDrv = if ! isNull tests
        then testSuite {
          name = tests.name or "${name}-test";
          srcs =
            ( # testSuite does run implFilter as well
              filteredSrcs ++ (tests.srcs or []));
          deps = filteredDeps ++ (tests.deps or []);
          expression = tests.expression;
          inherit impl;
        }
        else null;
    in lib.fix (self: runCommandNoCC "${name}" {
      nativeBuildInputs = [ makeWrapper ];
      LD_LIBRARY_PATH = libPath;
      LANG = "C.UTF-8";
      passthru = passthru // {
        lispName = name;
        lispDeps = [ selfLib ];
        lispNativeDeps = native;
        lispBinary = true;
        tests = testDrv;
      };
    } ''
      ${if ! isNull testDrv
        then "echo 'Test ${testDrv} succeeded'"
        else ""}
      mkdir -p $out/bin

      ${impl.runScript} ${
        impl.genDumpLisp {
          inherit name main;
          deps = ([ selfLib ] ++ lispDeps);
        }
      }

      wrapProgram $out/bin/${name} --prefix LD_LIBRARY_PATH : "${libPath}"
    '');

  # 'bundled' creates a "library" which makes a built-in package available,
  # such as any of SBCL's sb-* packages or ASDF. By default this is done
  # by calling 'require', but implementations are free to provide their
  # own specific bundled function.
  bundled = name:
    let
      # TODO(sterni): allow overriding args to underlying 'library' (e. g. srcs)
      defaultBundled = implementation: name: library {
        inherit name implementation;
        srcs = lib.singleton (builtins.toFile "${name}.lisp" "(require '${name})");
      };

      bundled' =
        { implementation ? defaultImplementation
        , name
        }:
        impls."${implementation}".bundled or (defaultBundled implementation) name;

    in (makeOverridable bundled') {
      inherit name;
    };

in {
  library = withExtras library;
  program = withExtras program;
  inherit bundled;

  # 'sbclWith' creates an image with the specified libraries /
  # programs loaded in SBCL.
  sbclWith = impls.sbcl.lispWith;
}