diff options
Diffstat (limited to 'nix')
51 files changed, 2595 insertions, 0 deletions
diff --git a/nix/binify/default.nix b/nix/binify/default.nix new file mode 100644 index 000000000000..d40930fd3334 --- /dev/null +++ b/nix/binify/default.nix @@ -0,0 +1,16 @@ +{ pkgs, lib, ... }: + +# Create a store path where the executable `exe` +# is linked to $out/bin/${name}. +# This is useful for e.g. including it as a “package” +# in `buildInputs` of a shell.nix. +# +# For example, if I have the exeutable /nix/store/…-hello, +# I can make it into /nix/store/…-binify-hello/bin/hello +# with `binify { exe = …; name = "hello" }`. +{ exe, name }: + +pkgs.runCommandLocal "${name}-bin" {} '' + mkdir -p $out/bin + ln -sT ${lib.escapeShellArg exe} $out/bin/${lib.escapeShellArg name} +'' diff --git a/nix/bufCheck/default.nix b/nix/bufCheck/default.nix new file mode 100644 index 000000000000..6d17cb460ecf --- /dev/null +++ b/nix/bufCheck/default.nix @@ -0,0 +1,9 @@ +# Check protobuf syntax and breaking. +# +{ depot, pkgs, ... }: + +pkgs.writeShellScriptBin "ci-buf-check" '' + ${depot.third_party.bufbuild}/bin/buf check lint --input "${depot.depotPath}" + # Report-only + ${depot.third_party.bufbuild}/bin/buf check breaking --input "${depot.depotPath}" --against-input "${depot.depotPath}/.git#branch=canon" || true +'' diff --git a/nix/buildGo/.skip-subtree b/nix/buildGo/.skip-subtree new file mode 100644 index 000000000000..8db1f814f653 --- /dev/null +++ b/nix/buildGo/.skip-subtree @@ -0,0 +1,2 @@ +Subdirectories of this folder should not be imported since they are +internal to buildGo.nix and incompatible with readTree. diff --git a/nix/buildGo/README.md b/nix/buildGo/README.md new file mode 100644 index 000000000000..37e0c06933f9 --- /dev/null +++ b/nix/buildGo/README.md @@ -0,0 +1,140 @@ +buildGo.nix +=========== + +This is an alternative [Nix][] build system for [Go][]. It supports building Go +libraries and programs, and even automatically generating Protobuf & gRPC +libraries. + +*Note:* This will probably end up being folded into [Nixery][]. + +## Background + +Most language-specific Nix tooling outsources the build to existing +language-specific build tooling, which essentially means that Nix ends up being +a wrapper around all sorts of external build systems. + +However, systems like [Bazel][] take an alternative approach in which the +compiler is invoked directly and the composition of programs and libraries stays +within a single homogeneous build system. + +Users don't need to learn per-language build systems and especially for +companies with large monorepo-setups ([like Google][]) this has huge +productivity impact. + +This project is an attempt to prove that Nix can be used in a similar style to +build software directly, rather than shelling out to other build systems. + +## Example + +Given a program layout like this: + +``` +. +├── lib <-- some library component +│ ├── bar.go +│ └── foo.go +├── api.proto <-- gRPC API definition +├── main.go <-- program implementation +└── default.nix <-- build instructions +``` + +The contents of `default.nix` could look like this: + +```nix +{ buildGo }: + +let + api = buildGo.grpc { + name = "someapi"; + proto = ./api.proto; + }; + + lib = buildGo.package { + name = "somelib"; + srcs = [ + ./lib/bar.go + ./lib/foo.go + ]; + }; +in buildGo.program { + name = "my-program"; + deps = [ api lib ]; + + srcs = [ + ./main.go + ]; +} +``` + +(If you don't know how to read Nix, check out [nix-1p][]) + +## Usage + +`buildGo` exposes five different functions: + +* `buildGo.program`: Build a Go binary out of the specified source files. + + | parameter | type | use | required? | + |-----------|-------------------------|------------------------------------------------|-----------| + | `name` | `string` | Name of the program (and resulting executable) | yes | + | `srcs` | `list<path>` | List of paths to source files | yes | + | `deps` | `list<drv>` | List of dependencies (i.e. other Go libraries) | no | + | `x_defs` | `attrs<string, string>` | Attribute set of linker vars (i.e. `-X`-flags) | no | + +* `buildGo.package`: Build a Go library out of the specified source files. + + | parameter | type | use | required? | + |-----------|--------------|------------------------------------------------|-----------| + | `name` | `string` | Name of the library | yes | + | `srcs` | `list<path>` | List of paths to source files | yes | + | `deps` | `list<drv>` | List of dependencies (i.e. other Go libraries) | no | + | `path` | `string` | Go import path for the resulting library | no | + +* `buildGo.external`: Build an externally defined Go library or program. + + This function performs analysis on the supplied source code (which + can use the standard Go tooling layout) and creates a tree of all + the packages contained within. + + This exists for compatibility with external libraries that were not + defined using buildGo. + + | parameter | type | use | required? | + |-----------|----------------|-----------------------------------------------|-----------| + | `path` | `string` | Go import path for the resulting package | yes | + | `src` | `path` | Path to the source **directory** | yes | + | `deps` | `list<drv>` | List of dependencies (i.e. other Go packages) | no | + + For some examples of how `buildGo.external` is used, check out + [`proto.nix`](./proto.nix). + +* `buildGo.proto`: Build a Go library out of the specified Protobuf definition. + + | parameter | type | use | required? | + |-------------|-------------|--------------------------------------------------|-----------| + | `name` | `string` | Name for the resulting library | yes | + | `proto` | `path` | Path to the Protobuf definition file | yes | + | `path` | `string` | Import path for the resulting Go library | no | + | `extraDeps` | `list<drv>` | Additional Go dependencies to add to the library | no | + +* `buildGo.grpc`: Build a Go library out of the specified gRPC definition. + + The parameters are identical to `buildGo.proto`. + +## Current status + +This project is work-in-progress. Crucially it is lacking the following features: + +* feature flag parity with Bazel's Go rules +* documentation building +* test execution + +There are still some open questions around how to structure some of those +features in Nix. + +[Nix]: https://nixos.org/nix/ +[Go]: https://golang.org/ +[Nixery]: https://github.com/google/nixery +[Bazel]: https://bazel.build/ +[like Google]: https://ai.google/research/pubs/pub45424 +[nix-1p]: https://github.com/tazjin/nix-1p diff --git a/nix/buildGo/default.nix b/nix/buildGo/default.nix new file mode 100644 index 000000000000..a2396dc3f770 --- /dev/null +++ b/nix/buildGo/default.nix @@ -0,0 +1,135 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 +# +# buildGo provides Nix functions to build Go packages in the style of Bazel's +# rules_go. + +{ pkgs ? import <nixpkgs> {} +, ... }: + +let + inherit (builtins) + attrNames + baseNameOf + dirOf + elemAt + filter + listToAttrs + map + match + readDir + replaceStrings + toString; + + inherit (pkgs) lib go runCommand fetchFromGitHub protobuf symlinkJoin; + + # Helpers for low-level Go compiler invocations + spaceOut = lib.concatStringsSep " "; + + includeDepSrc = dep: "-I ${dep}"; + includeSources = deps: spaceOut (map includeDepSrc deps); + + includeDepLib = dep: "-L ${dep}"; + includeLibs = deps: spaceOut (map includeDepLib deps); + + srcBasename = src: elemAt (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src)) 1; + srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}"; + srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs); + + allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps))); + + xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); + + pathToName = p: replaceStrings ["/"] ["_"] (toString p); + + # Add an `overrideGo` attribute to a function result that works + # similar to `overrideAttrs`, but is used specifically for the + # arguments passed to Go builders. + makeOverridable = f: orig: (f orig) // { + overrideGo = new: makeOverridable f (orig // (new orig)); + }; + + # High-level build functions + + # Build a Go program out of the specified files and dependencies. + program = { name, srcs, deps ? [], x_defs ? {} }: + let uniqueDeps = allDeps (map (d: d.gopkg) deps); + in runCommand name {} '' + ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs} + mkdir -p $out/bin + export GOROOT_FINAL=go + ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a + ''; + + # Build a Go library assembled out of the specified files. + # + # This outputs both the sources and compiled binary, as both are + # needed when downstream packages depend on it. + package = { name, srcs, deps ? [], path ? name, sfiles ? [] }: + let + uniqueDeps = allDeps (map (d: d.gopkg) deps); + + # The build steps below need to be executed conditionally for Go + # assembly if the analyser detected any *.s files. + # + # This is required for several popular packages (e.g. x/sys). + ifAsm = do: lib.optionalString (sfiles != []) do; + asmBuild = ifAsm '' + ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles} + ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles} + ''; + asmLink = ifAsm "-symabis ./symabis -asmhdr $out/go_asm.h"; + asmPack = ifAsm '' + ${go}/bin/go tool pack r $out/${path}.a ./asm.o + ''; + + gopkg = (runCommand "golib-${name}" {} '' + mkdir -p $out/${path} + ${srcList path (map (s: "${s}") srcs)} + ${asmBuild} + ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} + ${asmPack} + '') // { + inherit gopkg; + goDeps = uniqueDeps; + goImportPath = path; + }; + in gopkg; + + # Build a tree of Go libraries out of an external Go source + # directory that follows the standard Go layout and was not built + # with buildGo.nix. + # + # The derivation for each actual package will reside in an attribute + # named "gopkg", and an attribute named "gobin" for binaries. + external = import ./external { inherit pkgs program package; }; + + # Import support libraries needed for protobuf & gRPC support + protoLibs = import ./proto.nix { + inherit external; + }; + + # Build a Go library out of the specified protobuf definition. + proto = { name, proto, path ? name, goPackage ? name, extraDeps ? [] }: (makeOverridable package) { + inherit name path; + deps = [ protoLibs.goProto.proto.gopkg ] ++ extraDeps; + srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} '' + cp ${proto} ${baseNameOf proto} + ${protobuf}/bin/protoc --plugin=${protoLibs.goProto.protoc-gen-go.gopkg}/bin/protoc-gen-go \ + --go_out=plugins=grpc,import_path=${baseNameOf path}:. ${baseNameOf proto} + mv ./${goPackage}/*.pb.go $out + ''); + }; + + # Build a Go library out of the specified gRPC definition. + grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc.gopkg ]; }); + +in { + # Only the high-level builder functions are exposed, but made + # overrideable. + program = makeOverridable program; + package = makeOverridable package; + proto = makeOverridable proto; + grpc = makeOverridable grpc; + external = makeOverridable external; +} diff --git a/nix/buildGo/example/default.nix b/nix/buildGo/example/default.nix new file mode 100644 index 000000000000..99c0a7d79bd6 --- /dev/null +++ b/nix/buildGo/example/default.nix @@ -0,0 +1,47 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 + +# This file provides examples for how to use the various builder +# functions provided by `buildGo`. +# +# The features used in the example are not exhaustive, but should give +# users a quick introduction to how to use buildGo. + +let + buildGo = import ../default.nix {}; + + # Example use of buildGo.package, which creates an importable Go + # package from the specified source files. + examplePackage = buildGo.package { + name = "example"; + srcs = [ + ./lib.go + ]; + }; + + # Example use of buildGo.proto, which generates a Go library from a + # Protobuf definition file. + exampleProto = buildGo.proto { + name = "exampleproto"; + proto = ./thing.proto; + }; + + # Example use of buildGo.program, which builds an executable using + # the specified name and dependencies (which in turn must have been + # created via buildGo.package etc.) +in buildGo.program { + name = "example"; + + srcs = [ + ./main.go + ]; + + deps = [ + examplePackage + exampleProto + ]; + + x_defs = { + "main.Flag" = "successfully"; + }; +} diff --git a/nix/buildGo/example/lib.go b/nix/buildGo/example/lib.go new file mode 100644 index 000000000000..8a61370e994c --- /dev/null +++ b/nix/buildGo/example/lib.go @@ -0,0 +1,9 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 + +package example + +// UUID returns a totally random, carefully chosen UUID +func UUID() string { + return "3640932f-ad40-4bc9-b45d-f504a0f5910a" +} diff --git a/nix/buildGo/example/main.go b/nix/buildGo/example/main.go new file mode 100644 index 000000000000..bbcedbff8726 --- /dev/null +++ b/nix/buildGo/example/main.go @@ -0,0 +1,25 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 +// +// Package main provides a tiny example program for the Bazel-style +// Nix build system for Go. + +package main + +import ( + "example" + "exampleproto" + "fmt" +) + +var Flag string = "unsuccessfully" + +func main() { + thing := exampleproto.Thing{ + Id: example.UUID(), + KindOfThing: "test thing", + } + + fmt.Printf("The thing is a %s with ID %q\n", thing.Id, thing.KindOfThing) + fmt.Printf("The flag has been %s set\n", Flag) +} diff --git a/nix/buildGo/example/thing.proto b/nix/buildGo/example/thing.proto new file mode 100644 index 000000000000..0f6d6575e008 --- /dev/null +++ b/nix/buildGo/example/thing.proto @@ -0,0 +1,10 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; +package example; + +message Thing { + string id = 1; + string kind_of_thing = 2; +} diff --git a/nix/buildGo/external/default.nix b/nix/buildGo/external/default.nix new file mode 100644 index 000000000000..6f31fbe5caeb --- /dev/null +++ b/nix/buildGo/external/default.nix @@ -0,0 +1,95 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 +{ pkgs, program, package }: + +let + inherit (builtins) + elemAt + foldl' + fromJSON + head + length + listToAttrs + readFile + replaceStrings + tail + throw; + + inherit (pkgs) lib runCommand go jq ripgrep; + + pathToName = p: replaceStrings ["/"] ["_"] (toString p); + + # Collect all non-vendored dependencies from the Go standard library + # into a file that can be used to filter them out when processing + # dependencies. + stdlibPackages = runCommand "stdlib-pkgs.json" {} '' + export HOME=$PWD + export GOPATH=/dev/null + ${go}/bin/go list all | \ + ${ripgrep}/bin/rg -v 'vendor' | \ + ${jq}/bin/jq -R '.' | \ + ${jq}/bin/jq -c -s 'map({key: ., value: true}) | from_entries' \ + > $out + ''; + + analyser = program { + name = "analyser"; + + srcs = [ + ./main.go + ]; + + x_defs = { + "main.stdlibList" = "${stdlibPackages}"; + }; + }; + + mkset = path: value: + if path == [] then { gopkg = value; } + else { "${head path}" = mkset (tail path) value; }; + + last = l: elemAt l ((length l) - 1); + + toPackage = self: src: path: depMap: entry: + let + localDeps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) ( + throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'" + ) self) entry.localDeps; + + foreignDeps = map (d: lib.attrByPath [ d ] ( + throw "missing foreign dependency '${d}' in '${path}'" + ) depMap) entry.foreignDeps; + + args = { + srcs = map (f: src + ("/" + f)) entry.files; + deps = localDeps ++ foreignDeps; + }; + + libArgs = args // { + name = pathToName entry.name; + path = lib.concatStringsSep "/" ([ path ] ++ entry.locator); + sfiles = map (f: src + ("/" + f)) entry.sfiles; + }; + + binArgs = args // { + name = (last ((lib.splitString "/" path) ++ entry.locator)); + }; + in if entry.isCommand then (program binArgs) else (package libArgs); + +in { src, path, deps ? [] }: let + # Build a map of dependencies (from their import paths to their + # derivation) so that they can be conditionally imported only in + # sub-packages that require them. + depMap = listToAttrs (map (d: { + name = d.goImportPath; + value = d; + }) (map (d: d.gopkg) deps)); + + name = pathToName path; + analysisOutput = runCommand "${name}-structure.json" {} '' + ${analyser}/bin/analyser -path ${path} -source ${src} > $out + ''; + analysis = fromJSON (readFile analysisOutput); +in lib.fix(self: foldl' lib.recursiveUpdate {} ( + map (entry: mkset entry.locator (toPackage self src path depMap entry)) analysis +)) diff --git a/nix/buildGo/external/main.go b/nix/buildGo/external/main.go new file mode 100644 index 000000000000..50c6c8589519 --- /dev/null +++ b/nix/buildGo/external/main.go @@ -0,0 +1,190 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 + +// This tool analyses external (i.e. not built with `buildGo.nix`) Go +// packages to determine a build plan that Nix can import. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "go/build" + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + "strings" +) + +// Path to a JSON file describing all standard library import paths. +// This file is generated and set here by Nix during the build +// process. +var stdlibList string + +// pkg describes a single Go package within the specified source +// directory. +// +// Return information includes the local (relative from project root) +// and external (none-stdlib) dependencies of this package. +type pkg struct { + Name string `json:"name"` + Locator []string `json:"locator"` + Files []string `json:"files"` + SFiles []string `json:"sfiles"` + LocalDeps [][]string `json:"localDeps"` + ForeignDeps []string `json:"foreignDeps"` + IsCommand bool `json:"isCommand"` +} + +// findGoDirs returns a filepath.WalkFunc that identifies all +// directories that contain Go source code in a certain tree. +func findGoDirs(at string) ([]string, error) { + dirSet := make(map[string]bool) + + err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + name := info.Name() + // Skip folders that are guaranteed to not be relevant + if info.IsDir() && (name == "testdata" || name == ".git") { + return filepath.SkipDir + } + + // If the current file is a Go file, then the directory is popped + // (i.e. marked as a Go directory). + if !info.IsDir() && strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go") { + dirSet[filepath.Dir(path)] = true + } + + return nil + }) + + if err != nil { + return nil, err + } + + goDirs := []string{} + for k, _ := range dirSet { + goDirs = append(goDirs, k) + } + + return goDirs, nil +} + +// analysePackage loads and analyses the imports of a single Go +// package, returning the data that is required by the Nix code to +// generate a derivation for this package. +func analysePackage(root, source, importpath string, stdlib map[string]bool) (pkg, error) { + ctx := build.Default + ctx.CgoEnabled = false + + p, err := ctx.ImportDir(source, build.IgnoreVendor) + if err != nil { + return pkg{}, err + } + + local := [][]string{} + foreign := []string{} + + for _, i := range p.Imports { + if stdlib[i] { + continue + } + + if i == importpath { + local = append(local, []string{}) + } else if strings.HasPrefix(i, importpath+"/") { + local = append(local, strings.Split(strings.TrimPrefix(i, importpath+"/"), "/")) + } else { + foreign = append(foreign, i) + } + } + + prefix := strings.TrimPrefix(source, root+"/") + + locator := []string{} + if len(prefix) != len(source) { + locator = strings.Split(prefix, "/") + } else { + // Otherwise, the locator is empty since its the root package and + // no prefix should be added to files. + prefix = "" + } + + files := []string{} + for _, f := range p.GoFiles { + files = append(files, path.Join(prefix, f)) + } + + sfiles := []string{} + for _, f := range p.SFiles { + sfiles = append(sfiles, path.Join(prefix, f)) + } + + return pkg{ + Name: path.Join(importpath, prefix), + Locator: locator, + Files: files, + SFiles: sfiles, + LocalDeps: local, + ForeignDeps: foreign, + IsCommand: p.IsCommand(), + }, nil +} + +func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) { + f, err := ioutil.ReadFile(from) + if err != nil { + return + } + + err = json.Unmarshal(f, &pkgs) + return +} + +func main() { + source := flag.String("source", "", "path to directory with sources to process") + path := flag.String("path", "", "import path for the package") + + flag.Parse() + + if *source == "" { + log.Fatalf("-source flag must be specified") + } + + stdlibPkgs, err := loadStdlibPkgs(stdlibList) + if err != nil { + log.Fatalf("failed to load standard library index from %q: %s\n", stdlibList, err) + } + + goDirs, err := findGoDirs(*source) + if err != nil { + log.Fatalf("failed to walk source directory '%s': %s", *source, err) + } + + all := []pkg{} + for _, d := range goDirs { + analysed, err := analysePackage(*source, d, *path, stdlibPkgs) + + // If the Go source analysis returned "no buildable Go files", + // that directory should be skipped. + // + // This might be due to `+build` flags on the platform and other + // reasons (such as test files). + if _, ok := err.(*build.NoGoError); ok { + continue + } + + if err != nil { + log.Fatalf("failed to analyse package at %q: %s", d, err) + } + all = append(all, analysed) + } + + j, _ := json.Marshal(all) + fmt.Println(string(j)) +} diff --git a/nix/buildGo/proto.nix b/nix/buildGo/proto.nix new file mode 100644 index 000000000000..57e9a5ec98b0 --- /dev/null +++ b/nix/buildGo/proto.nix @@ -0,0 +1,84 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 +# +# This file provides derivations for the dependencies of a gRPC +# service in Go. + +{ external }: + +let + inherit (builtins) fetchGit map; +in rec { + goProto = external { + path = "github.com/golang/protobuf"; + src = fetchGit { + url = "https://github.com/golang/protobuf"; + rev = "ed6926b37a637426117ccab59282c3839528a700"; + }; + }; + + xnet = external { + path = "golang.org/x/net"; + + src = fetchGit { + url = "https://go.googlesource.com/net"; + rev = "ffdde105785063a81acd95bdf89ea53f6e0aac2d"; + }; + + deps = [ + xtext.secure.bidirule + xtext.unicode.bidi + xtext.unicode.norm + ]; + }; + + xsys = external { + path = "golang.org/x/sys"; + src = fetchGit { + url = "https://go.googlesource.com/sys"; + rev = "bd437916bb0eb726b873ee8e9b2dcf212d32e2fd"; + }; + }; + + xtext = external { + path = "golang.org/x/text"; + src = fetchGit { + url = "https://go.googlesource.com/text"; + rev = "cbf43d21aaebfdfeb81d91a5f444d13a3046e686"; + }; + }; + + genproto = external { + path = "google.golang.org/genproto"; + src = fetchGit { + url = "https://github.com/google/go-genproto"; + rev = "83cc0476cb11ea0da33dacd4c6354ab192de6fe6"; + }; + + deps = with goProto; [ + proto + ptypes.any + ]; + }; + + goGrpc = external { + path = "google.golang.org/grpc"; + deps = ([ + xnet.trace + xnet.http2 + xsys.unix + xnet.http2.hpack + genproto.googleapis.rpc.status + ] ++ (with goProto; [ + proto + ptypes + ptypes.duration + ptypes.timestamp + ])); + + src = fetchGit { + url = "https://github.com/grpc/grpc-go"; + rev = "d8e3da36ac481ef00e510ca119f6b68177713689"; + }; + }; +} diff --git a/nix/buildLisp/README.md b/nix/buildLisp/README.md new file mode 100644 index 000000000000..452f495debd4 --- /dev/null +++ b/nix/buildLisp/README.md @@ -0,0 +1,117 @@ +buildLisp.nix +============= + +This is a build system for Common Lisp, written in Nix. + +It aims to offer an alternative to ASDF for users who live in a +Nix-based ecosystem. This offers several advantages over ASDF: + +* Simpler (logic-less) package definitions +* Easy linking of native dependencies (from Nix) +* Composability with Nix tooling for other languages +* Effective, per-system caching strategies +* Easy overriding of dependencies and whatnot +* ... and more! + +The project is still in its early stages and some important +restrictions should be highlighted: + +* Only SBCL is supported (though the plan is to add support for at + least ABCL and Clozure CL, and maybe make it extensible) + +## Usage + +`buildLisp` exposes four different functions: + +* `buildLisp.library`: Builds a collection of Lisp files into a library. + + | parameter | type | use | required? | + |-----------|--------------|-------------------------------|-----------| + | `name` | `string` | Name of the library | yes | + | `srcs` | `list<path>` | List of paths to source files | yes | + | `deps` | `list<drv>` | List of dependencies | no | + | `native` | `list<drv>` | List of native dependencies | no | + | `test` | see "Tests" | Specification for test suite | no | + + The output of invoking this is a directory containing a FASL file + that is the concatenated result of all compiled sources. + +* `buildLisp.program`: Builds an executable program out of Lisp files. + + | parameter | type | use | required? | + |-----------|--------------|-------------------------------|-----------| + | `name` | `string` | Name of the program | yes | + | `srcs` | `list<path>` | List of paths to source files | yes | + | `deps` | `list<drv>` | List of dependencies | no | + | `native` | `list<drv>` | List of native dependencies | no | + | `main` | `string` | Entrypoint function | no | + | `test` | see "Tests" | Specification for test suite | no | + + The `main` parameter should be the name of a function and defaults + to `${name}:main` (i.e. the *exported* `main` function of the + package named after the program). + + The output of invoking this is a directory containing a + `bin/${name}`. + +* `buildLisp.bundled`: Creates a virtual dependency on a built-in library. + + Certain libraries ship with Lisp implementations, for example + UIOP/ASDF are commonly included but many implementations also ship + internals (such as SBCLs various `sb-*` libraries). + + This function takes a single string argument that is the name of a + built-in library and returns a "package" that simply requires this + library. + +* `buildLisp.sbclWith`: Creates an SBCL pre-loaded with various dependencies. + + This function takes a single argument which is a list of Lisp + libraries programs or programs. It creates an SBCL that is + pre-loaded with all of that Lisp code and can be used as the host + for e.g. Sly or SLIME. + +## Tests + +Both `buildLisp.library` and `buildLisp.program` take an optional argument +`tests`, which has the following supported fields: + + | parameter | type | use | required? | + |--------------|--------------|-------------------------------|-----------| + | `name` | `string` | Name of the test suite | no | + | `expression` | `string` | Lisp expression to run tests | yes | + | `srcs` | `list<path>` | List of paths to source files | no | + | `native` | `list<drv>` | List of native dependencies | no | + +the `expression` parameter should be a Lisp expression and will be evaluated +after loading all sources and dependencies (including library/program +dependencies). It must return a non-`NIL` value if the test suite has passed. + +## Example + +Using buildLisp could look like this: + +```nix +{ buildLisp, lispPkgs }: + +let libExample = buildLisp.library { + name = "lib-example"; + srcs = [ ./lib.lisp ]; + + deps = with lispPkgs; [ + (buildLisp.bundled "sb-posix") + iterate + cl-ppcre + ]; +}; +in buildLisp.program { + name = "example"; + deps = [ libExample ]; + srcs = [ ./main.lisp ]; + tests = { + deps = [ lispPkgs.fiveam ]; + srcs = [ ./tests.lisp ]; + expression = "(fiveam:run!)"; + }; +} +``` diff --git a/nix/buildLisp/default.nix b/nix/buildLisp/default.nix new file mode 100644 index 000000000000..9538f085299f --- /dev/null +++ b/nix/buildLisp/default.nix @@ -0,0 +1,258 @@ +# buildLisp provides Nix functions to build Common Lisp packages, +# targeting SBCL. +# +# buildLisp is designed to enforce conventions and do away with the +# free-for-all of existing Lisp build systems. + +{ pkgs ? import <nixpkgs> {}, ... }: + +let + inherit (builtins) map elemAt match filter; + inherit (pkgs) lib runCommandNoCC makeWrapper writeText writeShellScriptBin sbcl; + + # + # Internal helper definitions + # + + # 'genLoadLisp' generates Lisp code that instructs SBCL to load all + # the provided Lisp libraries. + genLoadLisp = deps: lib.concatStringsSep "\n" + (map (lib: "(load \"${lib}/${lib.lispName}.fasl\")") (allDeps deps)); + + # 'genCompileLisp' generates a Lisp file that instructs SBCL to + # compile the provided list of Lisp source files to $out. + genCompileLisp = srcs: deps: writeText "compile.lisp" '' + ;; This file compiles the specified sources into the Nix build + ;; directory, creating one FASL file for each source. + (require 'sb-posix) + + ${genLoadLisp deps} + + (defun nix-compile-lisp (file srcfile) + (let ((outfile (make-pathname :type "fasl" + :directory (or (sb-posix:getenv "NIX_BUILD_TOP") + (error "not running in a Nix build")) + :name (substitute #\- #\/ srcfile)))) + (multiple-value-bind (_outfile _warnings-p failure-p) + (compile-file srcfile :output-file outfile) + (if failure-p (sb-posix:exit 1) + (progn + ;; For the case of multiple files belonging to the same + ;; library being compiled, load them in order: + (load outfile) + + ;; Write them to the FASL list in the same order: + (format file "cat ~a~%" (namestring outfile))))))) + + (let ((*compile-verbose* t) + ;; FASL files are compiled into the working directory of the + ;; build and *then* moved to the correct out location. + (pwd (sb-posix:getcwd))) + + (with-open-file (file "cat_fasls" + :direction :output + :if-does-not-exist :create) + + ;; These forms were inserted by the Nix build: + ${ + lib.concatStringsSep "\n" (map (src: "(nix-compile-lisp file \"${src}\")") srcs) + } + )) + ''; + + # 'genTestLisp' generates a Lisp file that loads all sources and deps and + # executes expression + genTestLisp = name: srcs: deps: expression: writeText "${name}.lisp" '' + ;; Dependencies + ${genLoadLisp deps} + + ;; Sources + ${lib.concatStringsSep "\n" (map (src: "(load \"${src}\")") srcs)} + + ;; Test expression + (unless ${expression} + (exit :code 1)) + ''; + + # 'dependsOn' determines whether Lisp library 'b' depends on 'a'. + dependsOn = a: b: builtins.elem a b.lispDeps; + + # 'allDeps' flattens the list of dependencies (and their + # dependencies) into one ordered list of unique deps. + allDeps = deps: (lib.toposort dependsOn (lib.unique ( + lib.flatten (deps ++ (map (d: d.lispDeps) deps)) + ))).result; + + # 'allNative' extracts all native dependencies of a dependency list + # to ensure that library load paths are set correctly during all + # compilations and program assembly. + allNative = native: deps: lib.unique ( + lib.flatten (native ++ (map (d: d.lispNativeDeps) deps)) + ); + + # 'genDumpLisp' generates a Lisp file that instructs SBCL to dump + # the currently loaded image as an executable to $out/bin/$name. + # + # TODO(tazjin): Compression is currently unsupported because the + # SBCL in nixpkgs is, by default, not compiled with zlib support. + genDumpLisp = name: main: deps: writeText "dump.lisp" '' + (require 'sb-posix) + + ${genLoadLisp deps} + + (let* ((bindir (concatenate 'string (sb-posix:getenv "out") "/bin")) + (outpath (make-pathname :name "${name}" + :directory bindir))) + (save-lisp-and-die outpath + :executable t + :toplevel (function ${main}) + :purify t)) + ;; + ''; + + # Add an `overrideLisp` attribute to a function result that works + # similar to `overrideAttrs`, but is used specifically for the + # arguments passed to Lisp builders. + makeOverridable = f: orig: (f orig) // { + overrideLisp = new: makeOverridable f (orig // (new orig)); + }; + + # 'testSuite' builds a Common Lisp test suite that loads all of srcs and deps, + # and then executes expression to check its result + testSuite = { name, expression, srcs, deps ? [], native ? [] }: + let + lispNativeDeps = allNative native deps; + lispDeps = allDeps deps; + in runCommandNoCC name { + LD_LIBRARY_PATH = lib.makeLibraryPath lispNativeDeps; + LANG = "C.UTF-8"; + } '' + echo "Running test suite ${name}" + + ${sbcl}/bin/sbcl --script ${genTestLisp name srcs deps expression} \ + | tee $out + + echo "Test suite ${name} succeeded" + ''; + + # + # Public API functions + # + + # 'library' builds a list of Common Lisp files into a single FASL + # which can then be loaded into SBCL. + library = + { name + , srcs + , deps ? [] + , native ? [] + , tests ? null + }: + let + lispNativeDeps = (allNative native deps); + lispDeps = allDeps deps; + testDrv = if ! isNull tests + then testSuite { + name = tests.name or "${name}-test"; + srcs = srcs ++ (tests.srcs or []); + deps = deps ++ (tests.deps or []); + expression = tests.expression; + } + else null; + in lib.fix (self: runCommandNoCC "${name}-cllib" { + LD_LIBRARY_PATH = lib.makeLibraryPath lispNativeDeps; + LANG = "C.UTF-8"; + } '' + ${if ! isNull testDrv + then "echo 'Test ${testDrv} succeeded'" + else "echo 'No tests run'"} + ${sbcl}/bin/sbcl --script ${genCompileLisp srcs lispDeps} + + echo "Compilation finished, assembling FASL files" + + # FASL files can be combined by simply concatenating them + # together, but it needs to be in the compilation order. + mkdir $out + + chmod +x cat_fasls + ./cat_fasls > $out/${name}.fasl + '' // { + inherit lispNativeDeps lispDeps; + lispName = name; + lispBinary = false; + tests = testDrv; + sbcl = sbclWith [ self ]; + }); + + # 'program' creates an executable containing a dumped image of the + # specified sources and dependencies. + program = + { name + , main ? "${name}:main" + , srcs + , deps ? [] + , native ? [] + , tests ? null + }: + let + lispDeps = allDeps deps; + libPath = lib.makeLibraryPath (allNative native lispDeps); + selfLib = library { + inherit name srcs native; + deps = lispDeps; + }; + testDrv = if ! isNull tests + then testSuite { + name = tests.name or "${name}-test"; + srcs = + ( + srcs ++ (tests.srcs or [])); + deps = deps ++ (tests.deps or []); + expression = tests.expression; + } + else null; + in lib.fix (self: runCommandNoCC "${name}" { + nativeBuildInputs = [ makeWrapper ]; + LD_LIBRARY_PATH = libPath; + LANG = "C.UTF-8"; + } '' + ${if ! isNull testDrv + then "echo 'Test ${testDrv} succeeded'" + else ""} + mkdir -p $out/bin + + ${sbcl}/bin/sbcl --script ${ + genDumpLisp name main ([ selfLib ] ++ lispDeps) + } + + wrapProgram $out/bin/${name} --prefix LD_LIBRARY_PATH : "${libPath}" + '' // { + lispName = name; + lispDeps = [ selfLib ] ++ (tests.deps or []); + lispNativeDeps = native; + lispBinary = true; + tests = testDrv; + sbcl = sbclWith [ self ]; + }); + + # 'bundled' creates a "library" that calls 'require' on a built-in + # package, such as any of SBCL's sb-* packages. + bundled = name: (makeOverridable library) { + inherit name; + srcs = lib.singleton (builtins.toFile "${name}.lisp" "(require '${name})"); + }; + + # 'sbclWith' creates an image with the specified libraries / + # programs loaded. + sbclWith = deps: + let lispDeps = filter (d: !d.lispBinary) (allDeps deps); + in writeShellScriptBin "sbcl" '' + export LD_LIBRARY_PATH=${lib.makeLibraryPath (allNative [] lispDeps)}; + exec ${sbcl}/bin/sbcl ${lib.optionalString (deps != []) "--load ${writeText "load.lisp" (genLoadLisp lispDeps)}"} $@ + ''; +in { + library = makeOverridable library; + program = makeOverridable program; + sbclWith = makeOverridable sbclWith; + bundled = makeOverridable bundled; +} diff --git a/nix/buildLisp/example/default.nix b/nix/buildLisp/example/default.nix new file mode 100644 index 000000000000..6a518e4964a1 --- /dev/null +++ b/nix/buildLisp/example/default.nix @@ -0,0 +1,32 @@ +{ depot, ... }: + +let + inherit (depot.nix) buildLisp; + + # Example Lisp library. + # + # Currently the `name` attribute is only used for the derivation + # itself, it has no practical implications. + libExample = buildLisp.library { + name = "lib-example"; + srcs = [ + ./lib.lisp + ]; + }; + +# Example Lisp program. +# +# This builds & writes an executable for a program using the library +# above to disk. +# +# By default, buildLisp.program expects the entry point to be +# `$name:main`. This can be overridden by configuring the `main` +# attribute. +in buildLisp.program { + name = "example"; + deps = [ libExample ]; + + srcs = [ + ./main.lisp + ]; +} diff --git a/nix/buildLisp/example/lib.lisp b/nix/buildLisp/example/lib.lisp new file mode 100644 index 000000000000..e557de4ae5fd --- /dev/null +++ b/nix/buildLisp/example/lib.lisp @@ -0,0 +1,6 @@ +(defpackage lib-example + (:use :cl) + (:export :who)) +(in-package :lib-example) + +(defun who () "edef") diff --git a/nix/buildLisp/example/main.lisp b/nix/buildLisp/example/main.lisp new file mode 100644 index 000000000000..a29390cf4dba --- /dev/null +++ b/nix/buildLisp/example/main.lisp @@ -0,0 +1,7 @@ +(defpackage example + (:use :cl :lib-example) + (:export :main)) +(in-package :example) + +(defun main () + (format t "i <3 ~A~%" (who))) diff --git a/nix/buildTypedGo/default.nix b/nix/buildTypedGo/default.nix new file mode 100644 index 000000000000..f135b1ebb454 --- /dev/null +++ b/nix/buildTypedGo/default.nix @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# A crude wrapper around //nix/buildGo that supports the Go 2 alpha. +# +# The way the alpha is implemented is via a transpiler from typed to +# untyped Go. +{ depot, pkgs, ... }: + +let + inherit (builtins) + baseNameOf + stringLength + substring; + + inherit (depot.nix.buildGo) gpackage program; + + go2goext = file: substring 0 ((stringLength file) - 1) file; + go2go = file: pkgs.runCommandNoCC "${go2goext (baseNameOf file)}" {} '' + cp ${file} . + ${pkgs.go}/bin/go tool go2go translate *.go2 + mv *.go $out + ''; + +in rec { + program = { name, srcs, deps ? [], x_defs ? {} }: depot.nix.buildGo.program { + inherit name deps x_defs; + srcs = map go2go srcs; + }; + + package = { name, srcs, deps ? [], path ? name, sfiles ? [] }: depot.nix.buildGo.package { + inherit name deps path sfiles; + srcs = map go2go srcs; + }; +} diff --git a/nix/buildTypedGo/example/default.nix b/nix/buildTypedGo/example/default.nix new file mode 100644 index 000000000000..5b6d4171f99c --- /dev/null +++ b/nix/buildTypedGo/example/default.nix @@ -0,0 +1,8 @@ +{ depot, ... }: + +depot.nix.buildTypedGo.program { + name = "example"; + srcs = [ + ./main.go2 + ]; +} diff --git a/nix/buildTypedGo/example/main.go2 b/nix/buildTypedGo/example/main.go2 new file mode 100644 index 000000000000..8986f57b94c7 --- /dev/null +++ b/nix/buildTypedGo/example/main.go2 @@ -0,0 +1,15 @@ +package main + +import ( + "fmt" +) + +func Print(type T)(s []T) { + for _, v := range s { + fmt.Print(v) + } +} + +func main() { + Print([]string{"Hello, ", "TVL\n"}) +} diff --git a/nix/emptyDerivation/OWNERS b/nix/emptyDerivation/OWNERS new file mode 100644 index 000000000000..a742d0d22bf6 --- /dev/null +++ b/nix/emptyDerivation/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - Profpatsch diff --git a/nix/emptyDerivation/default.nix b/nix/emptyDerivation/default.nix new file mode 100644 index 000000000000..4165d4fd9ac1 --- /dev/null +++ b/nix/emptyDerivation/default.nix @@ -0,0 +1,20 @@ +{ depot, pkgs, ... }: + +let + emptyDerivation = import ./emptyDerivation.nix { + inherit pkgs; + inherit (pkgs) stdenv; + inherit (depot.nix) getBins; + }; + + tests = import ./tests.nix { + inherit emptyDerivation; + inherit pkgs; + inherit (depot.nix) writeExecline getBins; + inherit (depot.nix.runTestsuite) runTestsuite it assertEq; + }; + +in { + __functor = _: emptyDerivation; + inherit tests; +} diff --git a/nix/emptyDerivation/emptyDerivation.nix b/nix/emptyDerivation/emptyDerivation.nix new file mode 100644 index 000000000000..5e84abe2d505 --- /dev/null +++ b/nix/emptyDerivation/emptyDerivation.nix @@ -0,0 +1,34 @@ +{ stdenv, pkgs, getBins }: + +# The empty derivation. All it does is touch $out. +# Basically the unit value for derivations. +# +# In addition to simple test situations which require +# a derivation, we set __functor, so you can call it +# as a function and pass an attrset. The set you pass +# is `//`-merged with the attrset before calling derivation, +# so you can use this to add more fields. + +let + bins = getBins pkgs.s6-portable-utils [ "s6-touch" ] + // getBins pkgs.execline [ "importas" "exec" ]; + + emptiness = { + name = "empty-derivation"; + + # TODO(Profpatsch): can we get system from tvl? + inherit (stdenv) system; + + builder = bins.exec; + args = [ + bins.importas "out" "out" + bins.s6-touch "$out" + ]; + }; + +in (derivation emptiness) // { + # This allows us to call the empty derivation + # like a function and override fields/add new fields. + __functor = _: overrides: + derivation (emptiness // overrides); +} diff --git a/nix/emptyDerivation/tests.nix b/nix/emptyDerivation/tests.nix new file mode 100644 index 000000000000..053603b02772 --- /dev/null +++ b/nix/emptyDerivation/tests.nix @@ -0,0 +1,32 @@ +{ emptyDerivation, getBins, pkgs, writeExecline, runTestsuite, it, assertEq }: + +let + bins = getBins pkgs.s6-portable-utils [ "s6-echo" ]; + + empty = it "is just an empty path" [ + (assertEq "path empty" + (builtins.readFile emptyDerivation) + "") + ]; + + fooOut = emptyDerivation { + builder = writeExecline "foo-builder" {} [ + "importas" "out" "out" + "redirfd" "-w" "1" "$out" + bins.s6-echo "-n" "foo" + ]; + }; + + overrideBuilder = it "can override the builder" [ + (assertEq "output is foo" + (builtins.readFile fooOut) + "foo") + (assertEq "can add new drv variables" + (emptyDerivation { foo = "bar"; }).foo + "bar") + ]; + +in runTestsuite "emptyDerivation" [ + empty + overrideBuilder +] diff --git a/nix/escapeExecline/OWNERS b/nix/escapeExecline/OWNERS new file mode 100644 index 000000000000..a742d0d22bf6 --- /dev/null +++ b/nix/escapeExecline/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - Profpatsch diff --git a/nix/escapeExecline/default.nix b/nix/escapeExecline/default.nix new file mode 100644 index 000000000000..deef5c2c4ec8 --- /dev/null +++ b/nix/escapeExecline/default.nix @@ -0,0 +1,29 @@ +{ lib, ... }: +let + # replaces " and \ to \" and \\ respectively and quote with " + # e.g. + # a"b\c -> "a\"b\\c" + # a\"bc -> "a\\\"bc" + escapeExeclineArg = arg: + ''"${builtins.replaceStrings [ ''"'' ''\'' ] [ ''\"'' ''\\'' ] (toString arg)}"''; + + # Escapes an execline (list of execline strings) to be passed to execlineb + # Give it a nested list of strings. Nested lists are interpolated as execline + # blocks ({}). + # Everything is quoted correctly. + # + # Example: + # escapeExecline [ "if" [ "somecommand" ] "true" ] + # == ''"if" { "somecommand" } "true"'' + escapeExecline = execlineList: lib.concatStringsSep " " + (let + go = arg: + if builtins.isString arg then [(escapeExeclineArg arg)] + else if builtins.isPath arg then [(escapeExeclineArg "${arg}")] + else if lib.isDerivation arg then [(escapeExeclineArg arg)] + else if builtins.isList arg then [ "{" ] ++ builtins.concatMap go arg ++ [ "}" ] + else abort "escapeExecline can only hande nested lists of strings, was ${lib.generators.toPretty {} arg}"; + in builtins.concatMap go execlineList); + +in +escapeExecline diff --git a/nix/fetchGoModule/OWNERS b/nix/fetchGoModule/OWNERS new file mode 100644 index 000000000000..47b6a04183a2 --- /dev/null +++ b/nix/fetchGoModule/OWNERS @@ -0,0 +1,2 @@ +owners: + - edef diff --git a/nix/fetchGoModule/default.nix b/nix/fetchGoModule/default.nix new file mode 100644 index 000000000000..84794bbc50c7 --- /dev/null +++ b/nix/fetchGoModule/default.nix @@ -0,0 +1,32 @@ +{ lib, pkgs, ... }: + +let + + inherit (lib) + lowerChars + replaceStrings + upperChars + ; + + caseFold = replaceStrings upperChars (map (c: "!" + c) lowerChars); + +in + +{ path, version, sha256 }: + +pkgs.fetchurl { + name = "source"; + url = "https://proxy.golang.org/${caseFold path}/@v/v${version}.zip"; + inherit sha256; + + recursiveHash = true; + downloadToTemp = true; + + postFetch = '' + unpackDir="$TMPDIR/unpack" + mkdir "$unpackDir" + + ${pkgs.unzip}/bin/unzip "$downloadedFile" -d "$unpackDir" + mv "$unpackDir/${path}@v${version}" "$out" + ''; +} diff --git a/nix/getBins/OWNERS b/nix/getBins/OWNERS new file mode 100644 index 000000000000..a742d0d22bf6 --- /dev/null +++ b/nix/getBins/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - Profpatsch diff --git a/nix/getBins/default.nix b/nix/getBins/default.nix new file mode 100644 index 000000000000..5ba7584ed844 --- /dev/null +++ b/nix/getBins/default.nix @@ -0,0 +1,48 @@ +{ lib, pkgs, depot, ... }: + +# Takes a derivation and a list of binary names +# and returns an attribute set of `name -> path`. +# The list can also contain renames in the form of +# `{ use, as }`, which goes `as -> usePath`. +# +# It is usually used to construct an attrset `bins` +# containing all the binaries required in a file, +# similar to a simple import system. +# +# Example: +# +# bins = getBins pkgs.hello [ "hello" ] +# // getBins pkgs.coreutils [ "printf" "ln" "echo" ] +# // getBins pkgs.execline +# [ { use = "if"; as = "execlineIf" } ] +# // getBins pkgs.s6-portable-utils +# [ { use = "s6-test"; as = "test" } +# { use = "s6-cat"; as = "cat" } +# ]; +# +# provides +# bins.{hello,printf,ln,echo,execlineIf,test,cat} +# + +let + getBins = drv: xs: + let f = x: + # TODO(Profpatsch): typecheck + let x' = if builtins.isString x then { use = x; as = x; } else x; + in { + name = x'.as; + value = "${lib.getBin drv}/bin/${x'.use}"; + }; + in builtins.listToAttrs (builtins.map f xs); + + + tests = import ./tests.nix { + inherit getBins; + inherit (depot.nix) writeScriptBin; + inherit (depot.nix.runTestsuite) assertEq it runTestsuite; + }; + +in { + __functor = _: getBins; + inherit tests; +} diff --git a/nix/getBins/tests.nix b/nix/getBins/tests.nix new file mode 100644 index 000000000000..ff81deb5f1ec --- /dev/null +++ b/nix/getBins/tests.nix @@ -0,0 +1,40 @@ +{ writeScriptBin, assertEq, it, runTestsuite, getBins }: + +let + drv = writeScriptBin "hello" "it’s me"; + drv2 = writeScriptBin "goodbye" "tschau"; + + bins = getBins drv [ + "hello" + { use = "hello"; as = "also-hello"; } + ] + // getBins drv2 [ "goodbye" ] + ; + + simple = it "path is equal to the executable name" [ + (assertEq "path" + bins.hello + "${drv}/bin/hello") + (assertEq "content" + (builtins.readFile bins.hello) + "it’s me") + ]; + + useAs = it "use/as can be used to rename attributes" [ + (assertEq "path" + bins.also-hello + "${drv}/bin/hello") + ]; + + secondDrv = it "by merging attrsets you can build up bins" [ + (assertEq "path" + bins.goodbye + "${drv2}/bin/goodbye") + ]; + +in + runTestsuite "getBins" [ + simple + useAs + secondDrv + ] diff --git a/nix/readTree/README.md b/nix/readTree/README.md new file mode 100644 index 000000000000..c93cf2bfdd61 --- /dev/null +++ b/nix/readTree/README.md @@ -0,0 +1,81 @@ +readTree +======== + +This is a Nix program that builds up an attribute set tree for a large +repository based on the filesystem layout. + +It is in fact the tool that lays out the attribute set of this repository. + +As an example, consider a root (`.`) of a repository and a layout such as: + +``` +. +├── third_party +│ ├── default.nix +│ └── rustpkgs +│ ├── aho-corasick.nix +│ └── serde.nix +└── tools + ├── cheddar + │ └── default.nix + └── roquefort.nix +``` + +When `readTree` is called on that tree, it will construct an attribute set with +this shape: + +```nix +{ + tools = { + cheddar = ...; + roquefort = ...; + }; + + third_party = { + # the `default.nix` of this folder might have had arbitrary other + # attributes here, such as this: + favouriteColour = "orange"; + + rustpkgs = { + aho-corasick = ...; + serde = ...; + }; + }; +} +``` + +Every imported Nix file that yields an attribute set will have a `__readTree = +true;` attribute merged into it. + +## Traversal logic + +`readTree` will follow any subdirectories of a tree and import all Nix files, +with some exceptions: + +* A folder can declare that its children are off-limit by containing a + `.skip-subtree` file. Since the content of the file is not checked, it can be + useful to leave a note for a human in the file. +* If a folder contains a `default.nix` file, no *sibling* Nix files will be + imported - however children are traversed as normal. +* If a folder contains a `default.nix` it is loaded and, if it evaluates to a + set, *merged* with the children. If it evaluates to anything else the children + are *not traversed*. + +Traversal is lazy, `readTree` will only build up the tree as requested. This +currently has the downside that directories with no importable files end up in +the tree as empty nodes (`{}`). + +## Import structure + +`readTree` is called with two parameters: The arguments to pass to all imports, +and the initial path at which to start the traversal. + +The package headers in this repository follow the form `{ pkgs, ... }:` where +`pkgs` is a fixed-point of the entire package tree (see the `default.nix` at the +root of the depot). + +In theory `readTree` can pass arguments of different shapes, but I have found +this to be a good solution for the most part. + +Note that `readTree` does not currently make functions overridable, though it is +feasible that it could do that in the future. diff --git a/nix/readTree/default.nix b/nix/readTree/default.nix new file mode 100644 index 000000000000..dedef5724063 --- /dev/null +++ b/nix/readTree/default.nix @@ -0,0 +1,80 @@ +{ ... }: + +args: initPath: + +let + inherit (builtins) + attrNames + baseNameOf + concatStringsSep + filter + hasAttr + head + isAttrs + length + listToAttrs + map + match + readDir + substring; + + argsWithPath = parts: + let meta.locatedAt = parts; + in meta // (if isAttrs args then args else args meta); + + readDirVisible = path: + let + children = readDir path; + isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != "."; + names = filter isVisible (attrNames children); + in listToAttrs (map (name: { + inherit name; + value = children.${name}; + }) names); + + # Create a mark containing the location of this attribute. + marker = parts: { + __readTree = parts; + }; + + # The marker is added to every set that was imported directly by + # readTree. + importWithMark = path: parts: + let imported = import path (argsWithPath parts); + in if (isAttrs imported) + then imported // (marker parts) + else imported; + + nixFileName = file: + let res = match "(.*)\.nix" file; + in if res == null then null else head res; + + readTree = path: parts: + let + dir = readDirVisible path; + self = importWithMark path parts; + joinChild = c: path + ("/" + c); + + # Import subdirectories of the current one, unless the special + # `.skip-subtree` file exists which makes readTree ignore the + # children. + # + # This file can optionally contain information on why the tree + # should be ignored, but its content is not inspected by + # readTree + filterDir = f: dir."${f}" == "directory"; + children = if hasAttr ".skip-subtree" dir then [] else map (c: { + name = c; + value = readTree (joinChild c) (parts ++ [ c ]); + }) (filter filterDir (attrNames dir)); + + # Import Nix files + nixFiles = filter (f: f != null) (map nixFileName (attrNames dir)); + nixChildren = map (c: let p = joinChild (c + ".nix"); in { + name = c; + value = importWithMark p (parts ++ [ c ]); + }) nixFiles; + in if dir ? "default.nix" + then (if isAttrs self then self // (listToAttrs children) else self) + else (listToAttrs (nixChildren ++ children) // (marker parts)); +in readTree initPath [ (baseNameOf initPath) ] diff --git a/nix/runExecline/OWNERS b/nix/runExecline/OWNERS new file mode 100644 index 000000000000..a742d0d22bf6 --- /dev/null +++ b/nix/runExecline/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - Profpatsch diff --git a/nix/runExecline/default.nix b/nix/runExecline/default.nix new file mode 100644 index 000000000000..22d968a1c643 --- /dev/null +++ b/nix/runExecline/default.nix @@ -0,0 +1,19 @@ +{ depot, pkgs, lib, ... }: +let + runExecline = import ./runExecline.nix { + inherit (pkgs) stdenv; + inherit (depot.nix) escapeExecline getBins; + inherit pkgs lib; + }; + + tests = import ./tests.nix { + inherit runExecline; + inherit (depot.nix) getBins writeScript; + inherit (pkgs) stdenv coreutils; + inherit pkgs; + }; + +in { + __functor = _: runExecline; + inherit tests; +} diff --git a/nix/runExecline/runExecline.nix b/nix/runExecline/runExecline.nix new file mode 100644 index 000000000000..498e26e576f5 --- /dev/null +++ b/nix/runExecline/runExecline.nix @@ -0,0 +1,121 @@ +{ pkgs, stdenv, lib, getBins, escapeExecline }: + +# runExecline is a primitive building block +# for writing non-kitchen sink builders. +# +# It’s conceptually similar to `runCommand`, +# but instead of concatenating bash scripts left +# and right, it actually *uses* the features of +# `derivation`, passing things to `args` +# and making it possible to overwrite the `builder` +# in a sensible manner. +# +# Additionally, it provides a way to pass a nix string +# to `stdin` of the build script. +# +# Similar to //nix/writeExecline, the passed script is +# not a string, but a nested list of nix lists +# representing execline blocks. Escaping is +# done by the implementation, the user can just use +# normal nix strings. +# +# Example: +# +# runExecline "my-drv" { stdin = "hi!"; } [ +# "importas" "out" "out" +# # this pipes stdout of s6-cat to $out +# # and s6-cat redirects from stdin to stdout +# "redirfd" "-w" "1" "$out" bins.s6-cat +# ] +# +# which creates a derivation with "hi!" in $out. +# +# See ./tests.nix for more examples. + + +let + bins = getBins pkgs.execline [ + "execlineb" + { use = "if"; as = "execlineIf"; } + "redirfd" + "importas" + "exec" + ] + // getBins pkgs.s6-portable-utils [ + "s6-cat" + "s6-grep" + "s6-touch" + "s6-test" + "s6-chmod" + ]; + +in + +name: +{ +# a string to pass as stdin to the execline script +stdin ? "" +# a program wrapping the acutal execline invocation; +# should be in Bernstein-chaining style +, builderWrapper ? bins.exec +# additional arguments to pass to the derivation +, derivationArgs ? {} +}: +# the execline script as a nested list of string, +# representing the blocks; +# see docs of `escapeExecline`. +execline: + +# those arguments can’t be overwritten +assert !derivationArgs ? system; +assert !derivationArgs ? name; +assert !derivationArgs ? builder; +assert !derivationArgs ? args; + +derivation (derivationArgs // { + # TODO(Profpatsch): what about cross? + inherit (stdenv) system; + inherit name; + + # okay, `builtins.toFile` does not accept strings + # that reference drv outputs. This means we need + # to pass the script and stdin as envvar; + # this might clash with another passed envar, + # so we give it a long & unique name + _runExeclineScript = + let + in escapeExecline execline; + _runExeclineStdin = stdin; + passAsFile = [ + "_runExeclineScript" + "_runExeclineStdin" + ] ++ derivationArgs.passAsFile or []; + + # the default, exec acts as identity executable + builder = builderWrapper; + + args = [ + bins.importas # import script file as $script + "-ui" # drop the envvar afterwards + "script" # substitution name + "_runExeclineScriptPath" # passed script file + + bins.importas # do the same for $stdin + "-ui" + "stdin" + "_runExeclineStdinPath" + + bins.redirfd # now we + "-r" # read the file + "0" # into the stdin of execlineb + "$stdin" # that was given via stdin + + bins.execlineb # the actual invocation + # TODO(Profpatsch): depending on the use-case, -S0 might not be enough + # in all use-cases, then a wrapper for execlineb arguments + # should be added (-P, -S, -s). + "-S0" # set $@ inside the execline script + "-W" # die on syntax error + "$script" # substituted by importas + ]; +}) diff --git a/nix/runExecline/tests.nix b/nix/runExecline/tests.nix new file mode 100644 index 000000000000..a8f91f3cf3ae --- /dev/null +++ b/nix/runExecline/tests.nix @@ -0,0 +1,91 @@ +{ stdenv, pkgs, runExecline, getBins, writeScript +# https://www.mail-archive.com/skaware@list.skarnet.org/msg01256.html +, coreutils }: + +let + + bins = getBins coreutils [ "mv" ] + // getBins pkgs.execline [ + "execlineb" + { use = "if"; as = "execlineIf"; } + "redirfd" + "importas" + ] + // getBins pkgs.s6-portable-utils [ + "s6-chmod" + "s6-grep" + "s6-touch" + "s6-cat" + "s6-test" + ]; + + # execline block of depth 1 + block = args: builtins.map (arg: " ${arg}") args ++ [ "" ]; + + # derivation that tests whether a given line exists + # in the given file. Does not use runExecline, because + # that should be tested after all. + fileHasLine = line: file: derivation { + name = "run-execline-test-file-${file.name}-has-line"; + inherit (stdenv) system; + builder = bins.execlineIf; + args = + (block [ + bins.redirfd "-r" "0" file # read file to stdin + bins.s6-grep "-F" "-q" line # and grep for the line + ]) + ++ [ + # if the block succeeded, touch $out + bins.importas "-ui" "out" "out" + bins.s6-touch "$out" + ]; + preferLocalBuild = true; + allowSubstitutes = false; + }; + + # basic test that touches out + basic = runExecline "run-execline-test-basic" { + derivationArgs = { + preferLocalBuild = true; + allowSubstitutes = false; + }; + } [ + "importas" "-ui" "out" "out" + "${bins.s6-touch}" "$out" + ]; + + # whether the stdin argument works as intended + stdin = fileHasLine "foo" (runExecline "run-execline-test-stdin" { + stdin = "foo\nbar\nfoo"; + derivationArgs = { + preferLocalBuild = true; + allowSubstitutes = false; + }; + } [ + "importas" "-ui" "out" "out" + # this pipes stdout of s6-cat to $out + # and s6-cat redirects from stdin to stdout + "redirfd" "-w" "1" "$out" bins.s6-cat + ]); + + wrapWithVar = runExecline "run-execline-test-wrap-with-var" { + builderWrapper = writeScript "var-wrapper" '' + #!${bins.execlineb} -S0 + export myvar myvalue $@ + ''; + derivationArgs = { + preferLocalBuild = true; + allowSubstitutes = false; + }; + } [ + "importas" "-ui" "v" "myvar" + "if" [ bins.s6-test "myvalue" "=" "$v" ] + "importas" "out" "out" + bins.s6-touch "$out" + ]; + +in [ + basic + stdin + wrapWithVar +] diff --git a/nix/runTestsuite/default.nix b/nix/runTestsuite/default.nix new file mode 100644 index 000000000000..0105eb6fc946 --- /dev/null +++ b/nix/runTestsuite/default.nix @@ -0,0 +1,121 @@ +{ lib, pkgs, depot, ... }: + +# Run a nix testsuite. +# +# The tests are simple assertions on the nix level, +# and can use derivation outputs if IfD is enabled. +# +# You build a testsuite by bundling assertions into +# “it”s and then bundling the “it”s into a testsuite. +# +# Running the testsuite will abort evaluation if +# any assertion fails. +# +# Example: +# +# runTestsuite "myFancyTestsuite" [ +# (it "does an assertion" [ +# (assertEq "42 is equal to 42" "42" "42") +# (assertEq "also 23" 23 23) +# ]) +# (it "frmbls the brlbr" [ +# (assertEq true false) +# ]) +# ] +# +# will fail the second it group because true is not false. + +let + inherit (depot.nix.yants) sum struct string any unit defun list; + + # rewrite the builtins.partition result + # to use `ok` and `err` instead of `right` and `wrong`. + partitionTests = pred: xs: + let res = builtins.partition pred xs; + in { + ok = res.right; + err = res.wrong; + }; + + # The result of an assert, + # either it’s true (yep) or false (nope). + # If it’s nope, we return the left and right + # side of the assert, together with the description. + AssertResult = + sum "AssertResult" { + yep = struct "yep" { + test = string; + }; + nope = struct "nope" { + test = string; + left = any; + right = any; + }; + }; + + # Result of an it. An it is a bunch of asserts + # bundled up with a good description of what is tested. + ItResult = + struct "ItResult" { + it-desc = string; + asserts = list AssertResult; + }; + + # assert that left and right values are equal + assertEq = defun [ string any any AssertResult ] + (desc: left: right: + if left == right + then { yep = { test = desc; }; } + else { nope = { + test = desc; + inherit left right; + }; + }); + + # Annotate a bunch of asserts with a descriptive name + it = desc: asserts: { + it-desc = desc; + inherit asserts; + }; + + # Run a bunch of its and check whether all asserts are yep. + # If not, abort evaluation with `throw` + # and print the result of the test suite. + # + # Takes a test suite name as first argument. + runTestsuite = defun [ string (list ItResult) unit ] + (name: itResults: + let + goodAss = ass: { + good = AssertResult.match ass { + yep = _: true; + nope = _: false; + }; + x = ass; + }; + goodIt = it: { + inherit (it) it-desc; + asserts = partitionTests (ass: + AssertResult.match ass { + yep = _: true; + nope = _: false; + }) it.asserts; + }; + goodIts = partitionTests (it: (goodIt it).asserts.err == []); + res = goodIts itResults; + in + if res.err == [] + then {} + # TODO(Profpatsch): pretty printing of results + # and probably also somewhat easier to read output + else throw + ( "testsuite ${name} failed!\n" + + lib.generators.toPretty {} res)); + +in { + inherit + assertEq + it + runTestsuite + ; +} diff --git a/nix/tailscale/default.nix b/nix/tailscale/default.nix new file mode 100644 index 000000000000..a21af7d115dc --- /dev/null +++ b/nix/tailscale/default.nix @@ -0,0 +1,30 @@ +# This file defines a Nix helper function to create Tailscale ACL files. +# +# https://tailscale.com/kb/1018/install-acls + +{ depot, ... }: + +with depot.nix.yants; + +let + inherit (builtins) toFile toJSON; + + acl = struct "acl" { + Action = enum [ "accept" "reject" ]; + Users = list string; + Ports = list string; + }; + + acls = list entry; + + aclConfig = struct "aclConfig" { + # Static group mappings from group names to lists of users + Groups = option (attrs (list string)); + + # Hostname aliases to use in place of IPs + Hosts = option (attrs string); + + # Actual ACL entries + ACLs = list acl; + }; +in config: toFile "tailscale-acl.json" (toJSON (aclConfig config)) diff --git a/nix/writeExecline/OWNERS b/nix/writeExecline/OWNERS new file mode 100644 index 000000000000..a742d0d22bf6 --- /dev/null +++ b/nix/writeExecline/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - Profpatsch diff --git a/nix/writeExecline/default.nix b/nix/writeExecline/default.nix new file mode 100644 index 000000000000..8626aa46080f --- /dev/null +++ b/nix/writeExecline/default.nix @@ -0,0 +1,38 @@ +{ pkgs, depot, ... }: + +# Write an execline script, represented as nested nix lists. +# Everything is escaped correctly. +# https://skarnet.org/software/execline/ + +# TODO(Profpatsch) upstream into nixpkgs + +name: +{ + # "var": substitute readNArgs variables and start $@ + # from the (readNArgs+1)th argument + # "var-full": substitute readNArgs variables and start $@ from $0 + # "env": don’t substitute, set # and 0…n environment vaariables, where n=$# + # "none": don’t substitute or set any positional arguments + # "env-no-push": like "env", but bypass the push-phase. Not recommended. + argMode ? "var", + # Number of arguments to be substituted as variables (passed to "var"/"-s" or "var-full"/"-S" + readNArgs ? 0, +}: +# Nested list of lists of commands. +# Inner lists are translated to execline blocks. +argList: + +let + env = + if argMode == "var" then "s${toString readNArgs}" + else if argMode == "var-full" then "S${toString readNArgs}" + else if argMode == "env" then "" + else if argMode == "none" then "P" + else if argMode == "env-no-push" then "p" + else abort ''"${toString argMode}" is not a valid argMode, use one of "var", "var-full", "env", "none", "env-no-push".''; + +in + depot.nix.writeScript name '' + #!${pkgs.execline}/bin/execlineb -W${env} + ${depot.nix.escapeExecline argList} + '' diff --git a/nix/writeScript/default.nix b/nix/writeScript/default.nix new file mode 100644 index 000000000000..e8e6e0fa10ac --- /dev/null +++ b/nix/writeScript/default.nix @@ -0,0 +1,29 @@ +{ pkgs, depot, ... }: + +# Write the given string to $out +# and make it executable. + +let + bins = depot.nix.getBins pkgs.s6-portable-utils [ + "s6-cat" + "s6-chmod" + ]; + +in +name: +# string of the executable script that is put in $out +script: + +depot.nix.runExecline name { + stdin = script; + derivationArgs = { + preferLocalBuild = true; + allowSubstitutes = false; + }; +} [ + "importas" "out" "out" + # this pipes stdout of s6-cat to $out + # and s6-cat redirects from stdin to stdout + "if" [ "redirfd" "-w" "1" "$out" bins.s6-cat ] + bins.s6-chmod "0755" "$out" +] diff --git a/nix/writeScriptBin/default.nix b/nix/writeScriptBin/default.nix new file mode 100644 index 000000000000..ed26cf197e1e --- /dev/null +++ b/nix/writeScriptBin/default.nix @@ -0,0 +1,12 @@ +{ depot, ... }: + +# Like writeScript, +# but put the script into `$out/bin/${name}`. + +name: +script: + +depot.nix.binify { + exe = (depot.nix.writeScript name script); + inherit name; +} diff --git a/nix/yants/README.md b/nix/yants/README.md new file mode 100644 index 000000000000..d17ea61b52d1 --- /dev/null +++ b/nix/yants/README.md @@ -0,0 +1,88 @@ +yants +===== + +This is a tiny type-checker for data in Nix, written in Nix. + +# Features + +* Checking of primitive types (`int`, `string` etc.) +* Checking polymorphic types (`option`, `list`, `either`) +* Defining & checking struct/record types +* Defining & matching enum types +* Defining & matching sum types +* Defining function signatures (including curried functions) +* Types are composable! `option string`! `list (either int (option float))`! +* Type errors also compose! + +Currently lacking: + +* Any kind of inference +* Convenient syntax for attribute-set function signatures + +## Primitives & simple polymorphism + + + +## Structs + + + +## Nested structs! + + + +## Enums! + + + +## Functions! + + + +# Usage + +Yants can be imported from its `default.nix`. A single attribute (`lib`) can be +passed, which will otherwise be imported from `<nixpkgs>`. + +TIP: You do not need to clone my whole repository to use Yants! It is split out +into the `nix/yants` branch which you can clone with, for example, `git clone -b +nix/yants https://git.tazj.in yants`. + +Examples for the most common import methods would be: + +1. Import into scope with `with`: + ```nix + with (import ./default.nix {}); + # ... Nix code that uses yants ... + ``` + +2. Import as a named variable: + ```nix + let yants = import ./default.nix {}; + in yants.string "foo" # or other uses ... + ```` + +3. Overlay into `pkgs.lib`: + ```nix + # wherever you import your package set (e.g. from <nixpkgs>): + import <nixpkgs> { + overlays = [ + (self: super: { + lib = super.lib // { yants = import ./default.nix { inherit (super) lib; }; }; + }) + ]; + } + + # yants now lives at lib.yants, besides the other library functions! + ``` + +Please see my [Nix one-pager](https://github.com/tazjin/nix-1p) for more generic +information about the Nix language and what the above constructs mean. + +# Stability + +The current API of Yants is **not yet** considered stable, but it works fine and +should continue to do so even if used at an older version. + +Yants' tests use Nix versions above 2.2 - compatibility with older versions is +not guaranteed. diff --git a/nix/yants/default.nix b/nix/yants/default.nix new file mode 100644 index 000000000000..6da99fa3c8c4 --- /dev/null +++ b/nix/yants/default.nix @@ -0,0 +1,299 @@ +# Copyright 2019 Google LLC +# SPDX-License-Identifier: Apache-2.0 +# +# Provides a "type-system" for Nix that provides various primitive & +# polymorphic types as well as the ability to define & check records. +# +# All types (should) compose as expected. + +{ lib ? (import <nixpkgs> {}).lib, ... }: + +with builtins; let + prettyPrint = lib.generators.toPretty {}; + + # typedef' :: struct { + # name = string; + # checkType = function; (a -> result) + # checkToBool = option function; (result -> bool) + # toError = option function; (a -> result -> string) + # def = option any; + # match = option function; + # } -> type + # -> (a -> b) + # -> (b -> bool) + # -> (a -> b -> string) + # -> type + # + # This function creates an attribute set that acts as a type. + # + # It receives a type name, a function that is used to perform a + # check on an arbitrary value, a function that can translate the + # return of that check to a boolean that informs whether the value + # is type-conformant, and a function that can construct error + # messages from the check result. + # + # This function is the low-level primitive used to create types. For + # many cases the higher-level 'typedef' function is more appropriate. + typedef' = { name, checkType + , checkToBool ? (result: result.ok) + , toError ? (_: result: result.err) + , def ? null + , match ? null }: { + inherit name checkToBool toError; + + # check :: a -> bool + # + # This function is used to determine whether a given type is + # conformant. + check = value: checkToBool (checkType value); + + # checkType :: a -> struct { ok = bool; err = option string; } + # + # This function checks whether the passed value is type conformant + # and returns an optional type error string otherwise. + inherit checkType; + + # __functor :: a -> a + # + # This function checks whether the passed value is type conformant + # and throws an error if it is not. + # + # The name of this function is a special attribute in Nix that + # makes it possible to execute a type attribute set like a normal + # function. + __functor = self: value: + let result = self.checkType value; + in if checkToBool result then value + else throw (toError value result); + }; + + typeError = type: val: + "expected type '${type}', but value '${prettyPrint val}' is of type '${typeOf val}'"; + + # typedef :: string -> (a -> bool) -> type + # + # typedef is the simplified version of typedef' which uses a default + # error message constructor. + typedef = name: check: typedef' { + inherit name; + checkType = check; + checkToBool = r: r; + toError = value: _result: typeError name value; + }; + + checkEach = name: t: l: foldl' (acc: e: + let res = t.checkType e; + isT = t.checkToBool res; + in { + ok = acc.ok && isT; + err = if isT + then acc.err + else acc.err + "${prettyPrint e}: ${t.toError e res}\n"; + }) { ok = true; err = "expected type ${name}, but found:\n"; } l; +in lib.fix (self: { + # Primitive types + any = typedef "any" (_: true); + unit = typedef "unit" (v: v == {}); + int = typedef "int" isInt; + bool = typedef "bool" isBool; + float = typedef "float" isFloat; + string = typedef "string" isString; + path = typedef "path" (x: typeOf x == "path"); + drv = typedef "derivation" (x: isAttrs x && x ? "type" && x.type == "derivation"); + function = typedef "function" (x: isFunction x || (isAttrs x && x ? "__functor" + && isFunction x.__functor)); + + # Type for types themselves. Useful when defining polymorphic types. + type = typedef "type" (x: + isAttrs x + && hasAttr "name" x && self.string.check x.name + && hasAttr "checkType" x && self.function.check x.checkType + && hasAttr "checkToBool" x && self.function.check x.checkToBool + && hasAttr "toError" x && self.function.check x.toError + ); + + # Polymorphic types + option = t: typedef' rec { + name = "option<${t.name}>"; + checkType = v: + let res = t.checkType v; + in { + ok = isNull v || (self.type t).checkToBool res; + err = "expected type ${name}, but value does not conform to '${t.name}': " + + t.toError v res; + }; + }; + + eitherN = tn: typedef "either<${concatStringsSep ", " (map (x: x.name) tn)}>" + (x: any (t: (self.type t).check x) tn); + + either = t1: t2: self.eitherN [ t1 t2 ]; + + list = t: typedef' rec { + name = "list<${t.name}>"; + + checkType = v: if isList v + then checkEach name (self.type t) v + else { + ok = false; + err = typeError name v; + }; + }; + + attrs = t: typedef' rec { + name = "attrs<${t.name}>"; + + checkType = v: if isAttrs v + then checkEach name (self.type t) (attrValues v) + else { + ok = false; + err = typeError name v; + }; + }; + + # Structs / record types + # + # Checks that all fields match their declared types, no optional + # fields are missing and no unexpected fields occur in the struct. + # + # Anonymous structs are supported (e.g. for nesting) by omitting the + # name. + # + # TODO: Support open records? + struct = + # Struct checking is more involved than the simpler types above. + # To make the actual type definition more readable, several + # helpers are defined below. + let + # checkField checks an individual field of the struct against + # its definition and creates a typecheck result. These results + # are aggregated during the actual checking. + checkField = def: name: value: let result = def.checkType value; in rec { + ok = def.checkToBool result; + err = if !ok && isNull value + then "missing required ${def.name} field '${name}'\n" + else "field '${name}': ${def.toError value result}\n"; + }; + + # checkExtraneous determines whether a (closed) struct contains + # any fields that are not part of the definition. + checkExtraneous = def: has: acc: + if (length has) == 0 then acc + else if (hasAttr (head has) def) + then checkExtraneous def (tail has) acc + else checkExtraneous def (tail has) { + ok = false; + err = acc.err + "unexpected struct field '${head has}'\n"; + }; + + # checkStruct combines all structure checks and creates one + # typecheck result from them + checkStruct = def: value: + let + init = { ok = true; err = ""; }; + extraneous = checkExtraneous def (attrNames value) init; + + checkedFields = map (n: + let v = if hasAttr n value then value."${n}" else null; + in checkField def."${n}" n v) (attrNames def); + + combined = foldl' (acc: res: { + ok = acc.ok && res.ok; + err = if !res.ok then acc.err + res.err else acc.err; + }) init checkedFields; + in { + ok = combined.ok && extraneous.ok; + err = combined.err + extraneous.err; + }; + + struct' = name: def: typedef' { + inherit name def; + checkType = value: if isAttrs value + then (checkStruct (self.attrs self.type def) value) + else { ok = false; err = typeError name value; }; + + toError = _: result: "expected '${name}'-struct, but found:\n" + result.err; + }; + in arg: if isString arg then (struct' arg) else (struct' "anon" arg); + + # Enums & pattern matching + enum = + let + plain = name: def: typedef' { + inherit name def; + + checkType = (x: isString x && elem x def); + checkToBool = x: x; + toError = value: _: "'${prettyPrint value} is not a member of enum ${name}"; + }; + enum' = name: def: lib.fix (e: (plain name def) // { + match = x: actions: deepSeq (map e (attrNames actions)) ( + let + actionKeys = attrNames actions; + missing = foldl' (m: k: if (elem k actionKeys) then m else m ++ [ k ]) [] def; + in if (length missing) > 0 + then throw "Missing match action for members: ${prettyPrint missing}" + else actions."${e x}"); + }); + in arg: if isString arg then (enum' arg) else (enum' "anon" arg); + + # Sum types + # + # The representation of a sum type is an attribute set with only one + # value, where the key of the value denotes the variant of the type. + sum = + let + plain = name: def: typedef' { + inherit name def; + checkType = (x: + let variant = elemAt (attrNames x) 0; + in if isAttrs x && length (attrNames x) == 1 && hasAttr variant def + then let t = def."${variant}"; + v = x."${variant}"; + res = t.checkType v; + in if t.checkToBool res + then { ok = true; } + else { + ok = false; + err = "while checking '${name}' variant '${variant}': " + + t.toError v res; + } + else { ok = false; err = typeError name x; } + ); + }; + sum' = name: def: lib.fix (s: (plain name def) // { + match = x: actions: + let variant = deepSeq (s x) (elemAt (attrNames x) 0); + actionKeys = attrNames actions; + defKeys = attrNames def; + missing = foldl' (m: k: if (elem k actionKeys) then m else m ++ [ k ]) [] defKeys; + in if (length missing) > 0 + then throw "Missing match action for variants: ${prettyPrint missing}" + else actions."${variant}" x."${variant}"; + }); + in arg: if isString arg then (sum' arg) else (sum' "anon" arg); + + # Typed function definitions + # + # These definitions wrap the supplied function in type-checking + # forms that are evaluated when the function is called. + # + # Note that typed functions themselves are not types and can not be + # used to check values for conformity. + defun = + let + mkFunc = sig: f: { + inherit sig; + __toString = self: foldl' (s: t: "${s} -> ${t.name}") + "λ :: ${(head self.sig).name}" (tail self.sig); + __functor = _: f; + }; + + defun' = sig: func: if length sig > 2 + then mkFunc sig (x: defun' (tail sig) (func ((head sig) x))) + else mkFunc sig (x: ((head (tail sig)) (func ((head sig) x)))); + + in sig: func: if length sig < 2 + then (throw "Signature must at least have two types (a -> b)") + else defun' sig func; +}) diff --git a/nix/yants/screenshots/enums.png b/nix/yants/screenshots/enums.png new file mode 100644 index 000000000000..71673e7ab63c --- /dev/null +++ b/nix/yants/screenshots/enums.png Binary files differdiff --git a/nix/yants/screenshots/functions.png b/nix/yants/screenshots/functions.png new file mode 100644 index 000000000000..30ed50f8327b --- /dev/null +++ b/nix/yants/screenshots/functions.png Binary files differdiff --git a/nix/yants/screenshots/nested-structs.png b/nix/yants/screenshots/nested-structs.png new file mode 100644 index 000000000000..6b03ed65ceb7 --- /dev/null +++ b/nix/yants/screenshots/nested-structs.png Binary files differdiff --git a/nix/yants/screenshots/simple.png b/nix/yants/screenshots/simple.png new file mode 100644 index 000000000000..05a302cc6b9d --- /dev/null +++ b/nix/yants/screenshots/simple.png Binary files differdiff --git a/nix/yants/screenshots/structs.png b/nix/yants/screenshots/structs.png new file mode 100644 index 000000000000..fcbcf6415fad --- /dev/null +++ b/nix/yants/screenshots/structs.png Binary files differdiff --git a/nix/yants/tests/default.nix b/nix/yants/tests/default.nix new file mode 100644 index 000000000000..da539ca3562b --- /dev/null +++ b/nix/yants/tests/default.nix @@ -0,0 +1,95 @@ +{ depot, pkgs, ... }: + +with builtins; +with depot.nix.yants; + +# Note: Derivations are not included in the tests below as they cause +# issues with deepSeq. + +deepSeq rec { + # Test that all primitive types match + primitives = [ + (unit {}) + (int 15) + (bool false) + (float 13.37) + (string "Hello!") + (function (x: x * 2)) + (path /nix) + ]; + + # Test that polymorphic types work as intended + poly = [ + (option int null) + (list string [ "foo" "bar" ]) + (either int float 42) + ]; + + # Test that structures work as planned. + person = struct "person" { + name = string; + age = int; + + contact = option (struct { + email = string; + phone = option string; + }); + }; + + testPerson = person { + name = "Brynhjulf"; + age = 42; + contact.email = "brynhjulf@yants.nix"; + }; + + # Test enum definitions & matching + colour = enum "colour" [ "red" "blue" "green" ]; + testMatch = colour.match "red" { + red = "It is in fact red!"; + blue = throw "It should not be blue!"; + green = throw "It should not be green!"; + }; + + # Test sum type definitions + creature = sum "creature" { + human = struct { + name = string; + age = option int; + }; + + pet = enum "pet" [ "dog" "lizard" "cat" ]; + }; + + testSum = creature { + human = { + name = "Brynhjulf"; + age = 42; + }; + }; + + testSumMatch = creature.match testSum { + human = v: "It's a human named ${v.name}"; + pet = v: throw "It's not supposed to be a pet!"; + }; + + # Test curried function definitions + func = defun [ string int string ] + (name: age: "${name} is ${toString age} years old"); + + testFunc = func "Brynhjulf" 42; + + # Test that all types are types. + testTypes = map type [ + any bool drv float int string path + + (attrs int) + (eitherN [ int string bool ]) + (either int string) + (enum [ "foo" "bar" ]) + (list string) + (option int) + (option (list string)) + (struct { a = int; b = option string; }) + (sum { a = int; b = option string; }) + ]; +} (pkgs.writeText "yants-tests" "All tests passed!") |
