From b601d9a1b73effa9bd54286721443807533cf8e5 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sun, 26 Mar 2023 17:02:55 -0500 Subject: Initial commit --- .envrc | 2 ++ .gitignore | 1 + README.md | 31 +++++++++++++++++++++++++++++++ Tuprules.tup | 1 + nix_actor.nimble | 13 +++++++++++++ protocol.prs | 4 ++++ src/Tupfile | 2 ++ src/nix_actor.nim | 35 +++++++++++++++++++++++++++++++++++ src/nix_actor/Tupfile | 2 ++ src/nix_actor/main.nim | 7 +++++++ src/nix_actor/protocol.nim | 17 +++++++++++++++++ src/nix_actor/store.nim | 21 +++++++++++++++++++++ src/nix_actor/the_protocol.nim | 14 ++++++++++++++ 13 files changed, 150 insertions(+) create mode 100644 .envrc create mode 100644 .gitignore create mode 100644 README.md create mode 100644 Tuprules.tup create mode 100644 nix_actor.nimble create mode 100644 protocol.prs create mode 100644 src/Tupfile create mode 100644 src/nix_actor.nim create mode 100644 src/nix_actor/Tupfile create mode 100644 src/nix_actor/main.nim create mode 100644 src/nix_actor/protocol.nim create mode 100644 src/nix_actor/store.nim create mode 100644 src/nix_actor/the_protocol.nim diff --git a/.envrc b/.envrc new file mode 100644 index 000000000000..268a24bddc36 --- /dev/null +++ b/.envrc @@ -0,0 +1,2 @@ +source_env .. +use flake work#nix_actor \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000000..7ad627584e63 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/.direnv diff --git a/README.md b/README.md new file mode 100644 index 000000000000..cdb9ce6e1f2c --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +# Syndicated Nix Actor + +An actor for interacting with the [Nix](https://nixos.org/) daemon via the [Syndicated Actor Model](https://syndicate-lang.org/). + +*This is only a proof-of-concept and is not useful in any meaningful way.* + +## Example configuration +``` +; create and publish a dedicated dataspace +let ?nixspace = dataspace + + +$nixspace [ + ; request a build of nixpkgs#hello + ? [ + $log ! + ] +] + +; start nix_actor as a daemon +> + + +; hand-off a capablity to the Nix dataspace to the actor +? ?actor> [ + $actor +] +``` diff --git a/Tuprules.tup b/Tuprules.tup new file mode 100644 index 000000000000..bdc23e362698 --- /dev/null +++ b/Tuprules.tup @@ -0,0 +1 @@ +NIM_FLAGS += --backend:cpp diff --git a/nix_actor.nimble b/nix_actor.nimble new file mode 100644 index 000000000000..254222cb69e3 --- /dev/null +++ b/nix_actor.nimble @@ -0,0 +1,13 @@ +# Package + +version = "20230326" +author = "Emery Hemingway" +description = "Syndicated Nix Actor" +license = "Unlicense" +srcDir = "src" +bin = @["nix_actor"] + + +# Dependencies + +requires "nim >= 1.6.10", "syndicate >= 20230326" diff --git a/protocol.prs b/protocol.prs new file mode 100644 index 000000000000..8c7bd6a3b25e --- /dev/null +++ b/protocol.prs @@ -0,0 +1,4 @@ +version 1 . + +Build = . +Serve = . diff --git a/src/Tupfile b/src/Tupfile new file mode 100644 index 000000000000..1f7501b06749 --- /dev/null +++ b/src/Tupfile @@ -0,0 +1,2 @@ +include_rules +: nix_actor.nim | $(SYNDICATE_PROTOCOL) ./ |> !nim_bin |> diff --git a/src/nix_actor.nim b/src/nix_actor.nim new file mode 100644 index 000000000000..3aeba5b6fa4a --- /dev/null +++ b/src/nix_actor.nim @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +import std/[asyncdispatch, json, osproc] +import preserves, preserves/jsonhooks +import syndicate +from syndicate/protocols/dataspace import Observe +import ./nix_actor/protocol +import ./nix_actor/[main, store] + +type Observe = dataspace.Observe[Ref] + +proc build(spec: string): Build = + var execOutput = execProcess("nix", args = ["build", "--json", "--no-link", spec], options = {poUsePath}) + stderr.writeLine execOutput + var js = parseJson(execOutput) + Build(input: spec, output: js[0].toPreserve) + +proc bootNixFacet(ds: Ref; turn: var Turn): Facet = + # let store = openStore() + inFacet(turn) do (turn: var Turn): + let storePathObservation = ?Observe(pattern: !Build) ?? {0: grabLit()} + during(turn, ds, storePathObservation) do (spec: string): + stderr.writeLine "build ", spec + let a = build(spec) + discard publish(turn, ds, a) + +proc bootNixActor(root: Ref; turn: var Turn) = + connectStdio(root, turn) + during(turn, root, ?Serve) do (ds: Ref): + discard bootNixFacet(ds, turn) + +initNix() # Nix lib isn't actually being used but it's nice to know that it links. +bootDataspace("main", bootNixActor) +runForever() diff --git a/src/nix_actor/Tupfile b/src/nix_actor/Tupfile new file mode 100644 index 000000000000..530ef5039c59 --- /dev/null +++ b/src/nix_actor/Tupfile @@ -0,0 +1,2 @@ +include_rules +: ../../protocol.prs |> !preserves_schema_nim |> protocol.nim | ../ diff --git a/src/nix_actor/main.nim b/src/nix_actor/main.nim new file mode 100644 index 000000000000..072e97e589e5 --- /dev/null +++ b/src/nix_actor/main.nim @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +{.passC: staticExec("pkg-config --cflags nix-main").} +{.passL: staticExec("pkg-config --libs nix-main").} + +proc initNix*() {.importcpp: "nix::initNix", header: "shared.hh".} diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim new file mode 100644 index 000000000000..de667bc57732 --- /dev/null +++ b/src/nix_actor/protocol.nim @@ -0,0 +1,17 @@ + +import + std/typetraits, preserves + +type + Serve* {.preservesRecord: "serve".} = object + `cap`* {.preservesEmbedded.}: Preserve[void] + + Build* {.preservesRecord: "nix-build".} = object + `input`*: string + `output`*: Preserve[void] + +proc `$`*(x: Serve | Build): string = + `$`(toPreserve(x)) + +proc encode*(x: Serve | Build): seq[byte] = + encode(toPreserve(x)) diff --git a/src/nix_actor/store.nim b/src/nix_actor/store.nim new file mode 100644 index 000000000000..997720d54054 --- /dev/null +++ b/src/nix_actor/store.nim @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +{.passC: staticExec("pkg-config --cflags nix-store").} +{.passL: staticExec("pkg-config --libs nix-store").} + +{.passC: "'-DSYSTEM=\"x86_64-linux\"'".} + +type + StorePath {.importcpp: "nix::StorePath", header: "path.hh".} = object + discard + +proc isDerivation*(path: StorePath): bool {.importcpp.} + +type + Store {.importcpp: "nix::ref", header: "store-api.hh".} = object + discard + +proc ensurePath*(store: Store; path: StorePath) {.importcpp.} + +proc openStore*(): Store {.importcpp: "nix::openStore".} diff --git a/src/nix_actor/the_protocol.nim b/src/nix_actor/the_protocol.nim new file mode 100644 index 000000000000..d7216945c960 --- /dev/null +++ b/src/nix_actor/the_protocol.nim @@ -0,0 +1,14 @@ + +import + std/typetraits, preserves + +type + Build* {.preservesRecord: "nix-build".} = object + `input`*: string + `output`*: string + +proc `$`*(x: Build): string = + `$`(toPreserve(x)) + +proc encode*(x: Build): seq[byte] = + encode(toPreserve(x)) -- cgit 1.4.1 From 4e3e77171c12b1509b79975965d279fe9aca1dbc Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 30 May 2023 13:32:29 +0100 Subject: Add realise and eval --- .envrc | 2 +- README.md | 38 ++++++++++++++++--------------- Tuprules.tup | 3 +++ nix_actor.nimble | 22 ++++++++---------- protocol.prs | 5 ++++- src/Tupfile | 3 ++- src/nix_actor.nim | 56 ++++++++++++++++++++++++++++++++++++---------- src/nix_actor/protocol.nim | 16 ++++++++----- 8 files changed, 94 insertions(+), 51 deletions(-) diff --git a/.envrc b/.envrc index 268a24bddc36..6c4024627ec9 100644 --- a/.envrc +++ b/.envrc @@ -1,2 +1,2 @@ source_env .. -use flake work#nix_actor \ No newline at end of file +use flake syndicate#nix_actor diff --git a/README.md b/README.md index cdb9ce6e1f2c..653647e5ad75 100644 --- a/README.md +++ b/README.md @@ -6,26 +6,28 @@ An actor for interacting with the [Nix](https://nixos.org/) daemon via the [Synd ## Example configuration ``` -; create and publish a dedicated dataspace -let ?nixspace = dataspace - +? $nixspace [ -$nixspace [ - ; request a build of nixpkgs#hello - ? [ - $log ! - ] -] + ? [ ] -; start nix_actor as a daemon -> - + ? [] + ? [] -; hand-off a capablity to the Nix dataspace to the actor -? ?actor> [ - $actor + ? ?any [ + $log ! + ] + + $config [ + > + ? ?cap> [ + $cap { + dataspace: $nixspace + } + ] + + ] ] ``` diff --git a/Tuprules.tup b/Tuprules.tup index bdc23e362698..f28e9b75e233 100644 --- a/Tuprules.tup +++ b/Tuprules.tup @@ -1 +1,4 @@ +include ../syndicate-nim/depends.tup +NIM_FLAGS += --path:$(TUP_CWD)/../syndicate-nim/src + NIM_FLAGS += --backend:cpp diff --git a/nix_actor.nimble b/nix_actor.nimble index 254222cb69e3..af6bf4c09f6e 100644 --- a/nix_actor.nimble +++ b/nix_actor.nimble @@ -1,13 +1,9 @@ -# Package - -version = "20230326" -author = "Emery Hemingway" -description = "Syndicated Nix Actor" -license = "Unlicense" -srcDir = "src" -bin = @["nix_actor"] - - -# Dependencies - -requires "nim >= 1.6.10", "syndicate >= 20230326" +version = "20230530" +author = "Emery Hemingway" +description = "Syndicated Nix Actor" +license = "Unlicense" +srcDir = "src" +bin = @["nix_actor"] +backend = "cpp" + +requires "nim >= 1.6.10", "syndicate >= 20230530" diff --git a/protocol.prs b/protocol.prs index 8c7bd6a3b25e..beb5c13d9d97 100644 --- a/protocol.prs +++ b/protocol.prs @@ -1,4 +1,7 @@ version 1 . Build = . -Serve = . + +Realise = . + +Eval = . diff --git a/src/Tupfile b/src/Tupfile index 1f7501b06749..2036a765b33e 100644 --- a/src/Tupfile +++ b/src/Tupfile @@ -1,2 +1,3 @@ include_rules -: nix_actor.nim | $(SYNDICATE_PROTOCOL) ./ |> !nim_bin |> +: nix_actor.nim | $(SYNDICATE_PROTOCOL) ./ |> !nim_bin |> {bin} +: {bin} |> !assert_built |> diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 3aeba5b6fa4a..d257a7540a3c 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -1,35 +1,67 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -import std/[asyncdispatch, json, osproc] +import std/[asyncdispatch, json, osproc, strutils, tables] import preserves, preserves/jsonhooks import syndicate from syndicate/protocols/dataspace import Observe import ./nix_actor/protocol import ./nix_actor/[main, store] -type Observe = dataspace.Observe[Ref] +type + Value = Preserve[void] + Options = Table[Symbol, Value] + Observe = dataspace.Observe[Ref] proc build(spec: string): Build = var execOutput = execProcess("nix", args = ["build", "--json", "--no-link", spec], options = {poUsePath}) - stderr.writeLine execOutput var js = parseJson(execOutput) Build(input: spec, output: js[0].toPreserve) +proc realise(realise: Realise): seq[string] = + var execlines = execProcess("nix-store", args = ["--realize", realise.drv], options = {poUsePath}) + split(strip(execlines), '\n') + +proc eval(eval: Eval): Value = + var args = @["eval", "--json", "--expr", eval.expr] + for sym, val in eval.options: + add(args, "--" & $sym) + if not val.isString "": + var js: JsonNode + if fromPreserve(js, val): add(args, $js) + else: stderr.writeLine "invalid option ", sym, " ", val + var execOutput = strip execProcess("nix", args = args, options = {poUsePath}) + if execOutput != "": + var js = parseJson(execOutput) + result = js.toPreserve + proc bootNixFacet(ds: Ref; turn: var Turn): Facet = # let store = openStore() - inFacet(turn) do (turn: var Turn): - let storePathObservation = ?Observe(pattern: !Build) ?? {0: grabLit()} - during(turn, ds, storePathObservation) do (spec: string): - stderr.writeLine "build ", spec - let a = build(spec) - discard publish(turn, ds, a) + result = inFacet(turn) do (turn: var Turn): + + during(turn, ds, ?Observe(pattern: !Build) ?? {0: grabLit()}) do (spec: string): + discard publish(turn, ds, build(spec)) + + during(turn, ds, ?Observe(pattern: !Realise) ?? {0: grabLit()}) do (drvPath: string): + var ass = Realise(drv: drvPath) + ass.outputs = realise(ass) + discard publish(turn, ds, ass) + + during(turn, ds, ?Observe(pattern: !Eval) ?? {0: grabLit(), 1: grabDict()}) do (e: string, o: Value): + var ass = Eval(expr: e) + if not fromPreserve(ass.options, unpackLiterals(o)): + stderr.writeLine "invalid options ", o + else: + ass.result = eval(ass) + discard publish(turn, ds, ass) + +type Args {.preservesDictionary.} = object + dataspace: Ref proc bootNixActor(root: Ref; turn: var Turn) = connectStdio(root, turn) - during(turn, root, ?Serve) do (ds: Ref): + during(turn, root, ?Args) do (ds: Ref): discard bootNixFacet(ds, turn) initNix() # Nix lib isn't actually being used but it's nice to know that it links. -bootDataspace("main", bootNixActor) -runForever() +runActor("main", bootNixActor) diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index de667bc57732..1f3064e6aeca 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -1,17 +1,23 @@ import - std/typetraits, preserves + preserves, std/tables type - Serve* {.preservesRecord: "serve".} = object - `cap`* {.preservesEmbedded.}: Preserve[void] + Eval* {.preservesRecord: "eval".} = object + `expr`*: string + `options`*: Table[Symbol, Preserve[void]] + `result`*: Preserve[void] + + Realise* {.preservesRecord: "realise".} = object + `drv`*: string + `outputs`*: seq[string] Build* {.preservesRecord: "nix-build".} = object `input`*: string `output`*: Preserve[void] -proc `$`*(x: Serve | Build): string = +proc `$`*(x: Eval | Realise | Build): string = `$`(toPreserve(x)) -proc encode*(x: Serve | Build): seq[byte] = +proc encode*(x: Eval | Realise | Build): seq[byte] = encode(toPreserve(x)) -- cgit 1.4.1 From 5247d1f329d107fe7bbe2b842a907821de64356e Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 30 May 2023 23:27:11 +0100 Subject: Add instantiate and narinfo --- README.md | 6 ++-- protocol.prs | 7 +++++ src/nix_actor.nim | 73 +++++++++++++++++++++++++++++++++++++++------- src/nix_actor.nim.cfg | 1 + src/nix_actor/protocol.nim | 14 +++++++-- 5 files changed, 87 insertions(+), 14 deletions(-) create mode 100644 src/nix_actor.nim.cfg diff --git a/README.md b/README.md index 653647e5ad75..ac21e9414a85 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,15 @@ An actor for interacting with the [Nix](https://nixos.org/) daemon via the [Syndicated Actor Model](https://syndicate-lang.org/). -*This is only a proof-of-concept and is not useful in any meaningful way.* +*This is only a proof-of-concept and is not yet useful.* ## Example configuration ``` ? $nixspace [ - ? [ ] + ? {}; in pkgs.hello" { } ?drv> [ + ? [ ] + ] ? [] ? [] diff --git a/protocol.prs b/protocol.prs index beb5c13d9d97..7260783ad11d 100644 --- a/protocol.prs +++ b/protocol.prs @@ -1,7 +1,14 @@ version 1 . + Build = . Realise = . +Instantiate = . + Eval = . + +Narinfo = . + +Dict = {symbol: any ...:...} . diff --git a/src/nix_actor.nim b/src/nix_actor.nim index d257a7540a3c..fbf9458cad4b 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -import std/[asyncdispatch, json, osproc, strutils, tables] +import std/[asyncdispatch, httpclient, json, osproc, parseutils, strutils, tables] import preserves, preserves/jsonhooks import syndicate from syndicate/protocols/dataspace import Observe @@ -10,9 +10,48 @@ import ./nix_actor/[main, store] type Value = Preserve[void] - Options = Table[Symbol, Value] Observe = dataspace.Observe[Ref] +proc parseArgs(args: var seq[string]; opts: Dict) = + for sym, val in opts: + add(args, "--" & $sym) + if not val.isString "": + var js: JsonNode + if fromPreserve(js, val): add(args, $js) + else: stderr.writeLine "invalid option --", sym, " ", val + +proc parseNarinfo(info: var Dict; text: string) = + var + key, val: string + off: int + while off < len(text): + off = off + parseUntil(text, key, ':', off) + 1 + off = off + skipWhitespace(text, off) + off = off + parseUntil(text, val, '\n', off) + 1 + if key != "" and val != "": + if allCharsInSet(val, Digits): + info[Symbol key] = val.parsePreserves + else: + info[Symbol key] = val.toPreserve + +proc narinfo(turn: var Turn; ds: Ref; path: string) = + let + client = newAsyncHttpClient() + url = "https://cache.nixos.org/" & path & ".narinfo" + futGet = get(client, url) + stderr.writeLine "fetching ", url + addCallback(futGet, turn) do (turn: var Turn): + let resp = read(futGet) + if code(resp) != Http200: + close(client) + else: + let futBody = body(resp) + addCallback(futBody, turn) do (turn: var Turn): + close(client) + var narinfo = Narinfo(path: path) + parseNarinfo(narinfo.info, read(futBody)) + discard publish(turn, ds, narinfo) + proc build(spec: string): Build = var execOutput = execProcess("nix", args = ["build", "--json", "--no-link", spec], options = {poUsePath}) var js = parseJson(execOutput) @@ -22,15 +61,18 @@ proc realise(realise: Realise): seq[string] = var execlines = execProcess("nix-store", args = ["--realize", realise.drv], options = {poUsePath}) split(strip(execlines), '\n') +proc instantiate(instantiate: Instantiate): Value = + const cmd = "nix-instantiate" + var args = @["--expr", instantiate.expr] + parseArgs(args, instantiate.options) + var execOutput = strip execProcess(cmd, args = args, options = {poUsePath}) + execOutput.toPreserve + proc eval(eval: Eval): Value = - var args = @["eval", "--json", "--expr", eval.expr] - for sym, val in eval.options: - add(args, "--" & $sym) - if not val.isString "": - var js: JsonNode - if fromPreserve(js, val): add(args, $js) - else: stderr.writeLine "invalid option ", sym, " ", val - var execOutput = strip execProcess("nix", args = args, options = {poUsePath}) + const cmd = "nix" + var args = @["eval", "--expr", eval.expr] + parseArgs(args, eval.options) + var execOutput = strip execProcess(cmd, args = args, options = {poUsePath}) if execOutput != "": var js = parseJson(execOutput) result = js.toPreserve @@ -47,6 +89,14 @@ proc bootNixFacet(ds: Ref; turn: var Turn): Facet = ass.outputs = realise(ass) discard publish(turn, ds, ass) + during(turn, ds, ?Observe(pattern: !Instantiate) ?? {0: grabLit(), 1: grabDict()}) do (e: string, o: Value): + var ass = Instantiate(expr: e) + if not fromPreserve(ass.options, unpackLiterals(o)): + stderr.writeLine "invalid options ", o + else: + ass.result = instantiate(ass) + discard publish(turn, ds, ass) + during(turn, ds, ?Observe(pattern: !Eval) ?? {0: grabLit(), 1: grabDict()}) do (e: string, o: Value): var ass = Eval(expr: e) if not fromPreserve(ass.options, unpackLiterals(o)): @@ -55,6 +105,9 @@ proc bootNixFacet(ds: Ref; turn: var Turn): Facet = ass.result = eval(ass) discard publish(turn, ds, ass) + during(turn, ds, ?Observe(pattern: !Narinfo) ?? {0: grabLit()}) do (path: string): + narinfo(turn, ds, path) + type Args {.preservesDictionary.} = object dataspace: Ref diff --git a/src/nix_actor.nim.cfg b/src/nix_actor.nim.cfg new file mode 100644 index 000000000000..1f92ea59c700 --- /dev/null +++ b/src/nix_actor.nim.cfg @@ -0,0 +1 @@ +define:ssl diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 1f3064e6aeca..08989efc28a8 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -12,12 +12,22 @@ type `drv`*: string `outputs`*: seq[string] + Narinfo* {.preservesRecord: "narinfo".} = object + `path`*: string + `info`*: Dict + + Dict* = Table[Symbol, Preserve[void]] Build* {.preservesRecord: "nix-build".} = object `input`*: string `output`*: Preserve[void] -proc `$`*(x: Eval | Realise | Build): string = + Instantiate* {.preservesRecord: "instantiate".} = object + `expr`*: string + `options`*: Dict + `result`*: Preserve[void] + +proc `$`*(x: Eval | Realise | Narinfo | Dict | Build | Instantiate): string = `$`(toPreserve(x)) -proc encode*(x: Eval | Realise | Build): seq[byte] = +proc encode*(x: Eval | Realise | Narinfo | Dict | Build | Instantiate): seq[byte] = encode(toPreserve(x)) -- cgit 1.4.1 From 3c2225013a653221bb120e43688e603cc6f5f552 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 5 Jun 2023 17:12:06 +0100 Subject: Worker protocol passthru test --- nix_actor.nimble | 2 +- protocol.prs | 24 ++- src/nix_actor.nim | 19 +- src/nix_actor/protocol.nim | 71 ++++++- src/nix_actor/sockets.nim | 464 +++++++++++++++++++++++++++++++++++++++++ src/nix_actor/store.nim | 10 + src/nix_actor/the_protocol.nim | 14 -- 7 files changed, 579 insertions(+), 25 deletions(-) create mode 100644 src/nix_actor/sockets.nim delete mode 100644 src/nix_actor/the_protocol.nim diff --git a/nix_actor.nimble b/nix_actor.nimble index af6bf4c09f6e..c88a14265c98 100644 --- a/nix_actor.nimble +++ b/nix_actor.nimble @@ -1,4 +1,4 @@ -version = "20230530" +version = "20230607" author = "Emery Hemingway" description = "Syndicated Nix Actor" license = "Unlicense" diff --git a/protocol.prs b/protocol.prs index 7260783ad11d..4e437ec1dca2 100644 --- a/protocol.prs +++ b/protocol.prs @@ -1,6 +1,5 @@ version 1 . - Build = . Realise = . @@ -12,3 +11,26 @@ Eval = . Narinfo = . Dict = {symbol: any ...:...} . + +FieldInt = int . +FieldString = string . +Field = int / string . +Fields = [Field ...] . + +ActionStart = . +ActionStop = . +ActionResult = . + +; TODO: why not make target a singleton? +Missing = . + +PathInfo = . diff --git a/src/nix_actor.nim b/src/nix_actor.nim index fbf9458cad4b..443dc0b7d2ea 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -1,12 +1,12 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -import std/[asyncdispatch, httpclient, json, osproc, parseutils, strutils, tables] +import std/[asyncdispatch, httpclient, json, os, osproc, parseutils, strutils, tables] import preserves, preserves/jsonhooks import syndicate from syndicate/protocols/dataspace import Observe import ./nix_actor/protocol -import ./nix_actor/[main, store] +import ./nix_actor/[main, sockets] type Value = Preserve[void] @@ -39,7 +39,6 @@ proc narinfo(turn: var Turn; ds: Ref; path: string) = client = newAsyncHttpClient() url = "https://cache.nixos.org/" & path & ".narinfo" futGet = get(client, url) - stderr.writeLine "fetching ", url addCallback(futGet, turn) do (turn: var Turn): let resp = read(futGet) if code(resp) != Http200: @@ -108,13 +107,21 @@ proc bootNixFacet(ds: Ref; turn: var Turn): Facet = during(turn, ds, ?Observe(pattern: !Narinfo) ?? {0: grabLit()}) do (path: string): narinfo(turn, ds, path) -type Args {.preservesDictionary.} = object - dataspace: Ref +type + RefArgs {.preservesDictionary.} = object + dataspace: Ref + SocketArgs {.preservesDictionary.} = object + `listen-socket`: string proc bootNixActor(root: Ref; turn: var Turn) = connectStdio(root, turn) - during(turn, root, ?Args) do (ds: Ref): + during(turn, root, ?RefArgs) do (ds: Ref): discard bootNixFacet(ds, turn) + during(turn, root, ?SocketArgs) do (path: string): + removeFile(path) + asyncCheck(turn, emulateSocket(path)) + do: + removeFile(path) initNix() # Nix lib isn't actually being used but it's nice to know that it links. runActor("main", bootNixActor) diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 08989efc28a8..cf893f447e72 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -1,6 +1,6 @@ import - preserves, std/tables + preserves, std/sets, std/tables type Eval* {.preservesRecord: "eval".} = object @@ -12,22 +12,87 @@ type `drv`*: string `outputs`*: seq[string] + Missing* {.preservesRecord: "missing".} = object + `targets`*: HashSet[string] + `willBuild`*: HashSet[string] + `willSubstitute`*: HashSet[string] + `unknown`*: HashSet[string] + `downloadSize`*: BiggestInt + `narSize`*: BiggestInt + Narinfo* {.preservesRecord: "narinfo".} = object `path`*: string `info`*: Dict + FieldKind* {.pure.} = enum + `int`, `string` + `Field`* {.preservesOr.} = object + case orKind*: FieldKind + of FieldKind.`int`: + `int`*: int + + of FieldKind.`string`: + `string`*: string + + + PathInfo* {.preservesRecord: "path-info".} = object + `path`*: string + `deriver`*: string + `narHash`*: string + `references`*: HashSet[string] + `registrationTime`*: BiggestInt + `narSize`*: BiggestInt + `ultimate`*: bool + `sigs`*: HashSet[string] + `ca`*: string + Dict* = Table[Symbol, Preserve[void]] Build* {.preservesRecord: "nix-build".} = object `input`*: string `output`*: Preserve[void] + Fields* = seq[Field] + ActionStart* {.preservesRecord: "start".} = object + `id`*: BiggestInt + `level`*: BiggestInt + `type`*: BiggestInt + `text`*: string + `fields`*: Fields + `parent`*: BiggestInt + + FieldString* = string Instantiate* {.preservesRecord: "instantiate".} = object `expr`*: string `options`*: Dict `result`*: Preserve[void] -proc `$`*(x: Eval | Realise | Narinfo | Dict | Build | Instantiate): string = + FieldInt* = BiggestInt + ActionStop* {.preservesRecord: "stop".} = object + `id`*: BiggestInt + + ActionResult* {.preservesRecord: "result".} = object + `id`*: BiggestInt + `type`*: BiggestInt + `fields`*: Fields + +proc `$`*(x: Eval | Realise | Missing | Narinfo | Field | PathInfo | Dict | + Build | + Fields | + ActionStart | + FieldString | + Instantiate | + FieldInt | + ActionStop | + ActionResult): string = `$`(toPreserve(x)) -proc encode*(x: Eval | Realise | Narinfo | Dict | Build | Instantiate): seq[byte] = +proc encode*(x: Eval | Realise | Missing | Narinfo | Field | PathInfo | Dict | + Build | + Fields | + ActionStart | + FieldString | + Instantiate | + FieldInt | + ActionStop | + ActionResult): seq[byte] = encode(toPreserve(x)) diff --git a/src/nix_actor/sockets.nim b/src/nix_actor/sockets.nim new file mode 100644 index 000000000000..8aa9157682bd --- /dev/null +++ b/src/nix_actor/sockets.nim @@ -0,0 +1,464 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +import std/[asyncdispatch, asyncnet, os, sets, strtabs, strutils] +from std/nativesockets import AF_INET, AF_UNIX, SOCK_STREAM, Protocol + +import preserves +import ./protocol, ./store + +{.pragma: workerProtocol, importc, header: "worker-protocol.hh".} + +type Word = uint64 +proc `$`(w: Word): string = toHex(w) + +const + WORKER_MAGIC_1 = 0x6E697863 + WORKER_MAGIC_2 = 0x6478696F + PROTOCOL_VERSION = 256 or 35 + + STDERR_NEXT = 0x6F6C6d67 + STDERR_READ = 0x64617461 + STDERR_WRITE = 0x64617416 + STDERR_LAST = 0x616C7473 + STDERR_ERROR = 0x63787470 + STDERR_START_ACTIVITY = 0x53545254 + STDERR_STOP_ACTIVITY = 0x53544F50 + STDERR_RESULT = 0x52534C54 + + wopIsValidPath = 1 + wopHasSubstitutes = 3 + wopQueryReferrers = 6 + wopAddToStore = 7 + wopBuildPaths = 9 + wopEnsurePath = 10 + wopAddTempRoot = 11 + wopAddIndirectRoot = 12 + wopSyncWithGC = 13 + wopFindRoots = 14 + wopSetOptions = 19 + wopCollectGarbage = 20 + wopQuerySubstitutablePathInfo = 21 + wopQueryAllValidPaths = 23 + wopQueryFailedPaths = 24 + wopClearFailedPaths = 25 + wopQueryPathInfo = 26 + wopQueryPathFromHashPart = 29 + wopQuerySubstitutablePathInfos = 30 + wopQueryValidPaths = 31 + wopQuerySubstitutablePaths = 32 + wopQueryValidDerivers = 33 + wopOptimiseStore = 34 + wopVerifyStore = 35 + wopBuildDerivation = 36 + wopAddSignatures = 37 + wopNarFromPath = 38 + wopAddToStoreNar = 39 + wopQueryMissing = 40 + wopQueryDerivationOutputMap = 41 + wopRegisterDrvOutput = 42 + wopQueryRealisation = 43 + wopAddMultipleToStore = 44 + wopAddBuildLog = 45 + wopBuildPathsWithResults = 46 + +type + ProtocolError = object of IOError + Version = uint16 + Session = ref object + client, daemon: AsyncSocket + buffer: seq[Word] + version: Version + +func major(version: Version): uint16 = version and 0xff00 +func minor(version: Version): uint16 = version and 0x00ff + +proc daemonSocketPath: string = + getEnv( + "NIX_DAEMON_SOCKET_PATH", + "/nix/var/nix/daemon-socket/socket") + +proc send(session: Session; sock: AsyncSocket; words: varargs[Word]): Future[void] = + for i, word in words: session.buffer[i] = word + send(sock, addr session.buffer[0], words.len shl 3) + +proc send(session: Session; sock: AsyncSocket; s: string): Future[void] = + let wordCount = (s.len + 7) shr 3 + if wordCount > session.buffer.len: setLen(session.buffer, wordCount) + session.buffer[0] = Word s.len + if wordCount > 0: + session.buffer[wordCount] = 0x00 + copyMem(addr session.buffer[1], unsafeAddr s[0], s.len) + send(sock, addr session.buffer[0], (1 + wordCount) shl 3) + +proc recvWord(sock: AsyncSocket): Future[Word] {.async.} = + var w: Word + let n = await recvInto(sock, addr w, sizeof(Word)) + if n != sizeof(Word): raise newException(ProtocolError, "short read of word") + return w + +proc passWord(a, b: AsyncSocket): Future[Word] {.async.} = + var w = await recvWord(a) + await send(b, addr w, sizeof(Word)) + return w + +proc recvString(sock: AsyncSocket): Future[string] {.async.} = + let w = await recvWord(sock) + let stringLen = int w + var s: string + if stringLen > 0: + s.setLen((stringLen + 7) and (not 7)) + let n = await recvInto(sock, addr s[0], s.len) + if n != s.len: + raise newException(ProtocolError, "short string read") + setLen(s, stringLen) + return s + +proc passString(session: Session; a, b: AsyncSocket): Future[string] {.async.} = + var s = await recvString(a) + await send(session, b, s) + return s + +proc passStringSeq(session: Session; a, b: AsyncSocket): Future[seq[string]] {.async.} = + let count = int(await passWord(a, b)) + var strings = newSeq[string](count) + for i in 0..= 16 + info.ultimate = (await passDaemonWord(session)) != 0 + info.sigs = await passDaemonStringSet(session) + info.ca = await passDaemonString(session) + return info + +proc passChunks(session: Session; a, b: AsyncSocket): Future[int] {.async.} = + var total: int + while true: + let chunkLen = int(await passWord(a, b)) + if chunkLen == 0: + break + else: + let wordLen = (chunkLen + 7) shr 3 + if session.buffer.len < wordLen: setLen(session.buffer, wordLen) + let recvLen = await recvInto(a, addr session.buffer[0], chunkLen) + # each chunk must be recved contiguously + if recvLen != chunkLen: + raise newException(ProtocolError, "invalid chunk read") + await send(b, addr session.buffer[0], recvLen) + inc(total, recvLen) + return total + +proc passClientChunks(session: Session): Future[int] = + passChunks(session, session.client, session.daemon) + +proc passErrorDaemonError(session: Session) {.async.} = + let + typ = await passDaemonString(session) + assert typ == "Error" + let + lvl = await passDaemonWord(session) + name = await passDaemonString(session) + msg = passDaemonString(session) + havePos = await passDaemonWord(session) + assert havePos == 0 + let + nrTraces = await passDaemonWord(session) + for i in 1..nrTraces: + let havPos = await passDaemonWord(session) + assert havPos == 0 + let msg = await passDaemonString(session) + +proc passDaemonFields(session: Session): Future[Fields] {.async.} = + let count = await passDaemonWord(session) + var fields = newSeq[Field](count) + for i in 0..= 26 + await passErrorDaemonError(session) + + of STDERR_NEXT: + let s = await passDaemonString(session) + + of STDERR_START_ACTIVITY: + var act: ActionStart + act.id = BiggestInt(await passDaemonWord(session)) + act.level = BiggestInt(await passDaemonWord(session)) + act.`type` = BiggestInt(await passDaemonWord(session)) + act.text = await passDaemonString(session) + act.fields = await passDaemonFields(session) + act.parent = BiggestInt(await passDaemonWord(session)) + + of STDERR_STOP_ACTIVITY: + var act: ActionStop + act.id = BiggestInt(await passDaemonWord(session)) + + of STDERR_RESULT: + var act: ActionResult + act.id = BiggestInt(await passDaemonWord(session)) + act.`type` = BiggestInt(await passDaemonWord(session)) + act.fields = await passDaemonFields(session) + + of STDERR_LAST: + break + + else: + raise newException(ProtocolError, "unknown work verb " & $word) + +#[ +proc fromClient(miss: var Missing; socket: AsyncSocket) {.async.} = + result.targets = await passClientStringSet(session) + +proc fromDaemon(miss: var Missing; socket: AsyncSocket) {.async.} = + miss.willBuild = await passDaemonStringSet(session) + miss.willSubstitute = await passDaemonStringSet(session) + miss.unknown = await passDaemonStringSet(session) + miss.downloadSize = BiggestInt await passDaemonWord(session) + miss.narSize = BiggestInt await passDaemonWord(session) +]# + +proc loop(session: Session) {.async.} = + var chunksTotal: int + try: + while not session.client.isClosed: + let wop = await passClientWord(session) + case wop + of wopIsValidPath: + let path = await passClientString(session) + stderr.writeLine "wopIsValidPath ", path + await passWork(session) + let word = await passDaemonWord(session) + + of wopAddToStore: + assert session.version.minor >= 25 + let + name = await passClientString(session) + caMethod = await passClientString(session) + refs = await passClientStringSet(session) + repairBool = await passClientWord(session) + stderr.writeLine "wopAddToStore ", name + let n = await passClientChunks(session) + inc(chunksTotal, n) + await passWork(session) + let info = await passDaemonValidPathInfo(session, true) + + of wopAddTempRoot: + let path = await passClientString(session) + stderr.writeLine "wopAddTempRoot ", path + await passWork(session) + discard await passDaemonWord(session) + + of wopAddIndirectRoot: + let path = await passClientString(session) + stderr.writeLine "wopAddIndirectRoot ", path + await passWork(session) + discard await passDaemonWord(session) + + of wopSetOptions: + discard passClientWord(session) # keepFailed + discard passClientWord(session) # keepGoing + discard passClientWord(session) # tryFallback + discard passClientWord(session) # verbosity + discard passClientWord(session) # maxBuildJobs + discard passClientWord(session) # maxSilentTime + discard passClientWord(session) # useBuildHook + discard passClientWord(session) # verboseBuild + discard passClientWord(session) # logType + discard passClientWord(session) # printBuildTrace + discard passClientWord(session) # buildCores + discard passClientWord(session) # useSubstitutes + assert session.version.minor >= 12 + let overrides = await passClientStringMap(session) + await passWork(session) + + of wopQueryPathInfo: + assert session.version >= 17 + let path = await passClientString(session) + stderr.writeLine "wopQueryPathInfo ", path + await passWork(session) + let valid = await passDaemonWord(session) + if valid != 0: + var info = await passDaemonValidPathInfo(session, false) + info.path = path + stderr.writeLine "wopQueryPathInfo ", $info + + of wopQueryMissing: + assert session.version >= 30 + var miss: Missing + miss.targets = await passClientStringSet(session) + await passWork(session) + miss.willBuild = await passDaemonStringSet(session) + miss.willSubstitute = await passDaemonStringSet(session) + miss.unknown = await passDaemonStringSet(session) + miss.downloadSize = BiggestInt await passDaemonWord(session) + miss.narSize = BiggestInt await passDaemonWord(session) + stderr.writeLine "wopQueryMissing ", $miss + + of wopBuildPathsWithResults: + assert session.version >= 34 + let + drvs = await passClientStringSeq(session) + buildMode = await passClientWord(session) + stderr.writeLine "wopBuildPathsWithResults drvs ", $drvs + await passWork(session) + let count = await passDaemonWord(session) + for _ in 1..count: + let + path = await passDaemonString(session) + status = await passDaemonWord(session) + errorMsg = await passDaemonString(session) + timesBUild = await passDaemonWord(session) + isNonDeterministic = await passDaemonWord(session) + startTime = await passDaemonWord(session) + stopTime = await passDaemonWord(session) + outputs = await passDaemonStringMap(session) + + else: + stderr.writeLine "unknown worker op ", wop.int + break + except ProtocolError as err: + stderr.writeLine "connection terminated" + stderr.writeLine "chunk bytes transfered: ", formatSize(chunksTotal) + finally: + close(session.daemon) + close(session.client) + +proc handshake(listener: AsyncSocket): Future[Session] {.async.} = + ## Take the next connection from `listener` and return a `Session`. + let session = Session(buffer: newSeq[Word](1024)) # 8KiB + session.client = await listener.accept() + session.daemon = newAsyncSocket( + domain = AF_UNIX, + sockType = SOCK_STREAM, + protocol = cast[Protocol](0), + buffered = false) + await connectUnix(session.daemon, daemonSocketPath()) + let clientMagic = await passClientWord(session) + if clientMagic != WORKER_MAGIC_1: + raise newException(ProtocolError, "invalid protocol magic") + let daemonMagic = await passDaemonWord(session) + let daemonVersion = await passDaemonWord(session) + session.version = Version(await passClientWord(session)) + if session.version < 0x1_0a: + raise newException(ProtocolError, "obsolete protocol version") + assert session.version.minor >= 14 + discard await(passClientWord(session)) + # obsolete CPU affinity + assert session.version.minor >= 11 + discard await(passClientWord(session)) + # obsolete reserveSpace + assert session.version.minor >= 33 + let daemonVersionString = await passDaemonString(session) + assert daemonVersionString == $store.nixVersion + await passWork(session) + return session + +proc emulateSocket*(path: string) {.async, gcsafe.} = + let listener = newAsyncSocket( + domain = AF_UNIX, + sockType = SOCK_STREAM, + protocol = cast[Protocol](0), + buffered = false) + bindUnix(listener, path) + listen(listener) + stderr.writeLine "listening on ", path + while not listener.isClosed: + try: + let session = await handshake(listener) + assert not session.isNil + asyncCheck loop(session) + except ProtocolError as err: + stderr.writeLine "failed to service client, ", err.msg + +when isMainModule: + const path = "/tmp/worker.nix.socket" + if fileExists(path): removeFile(path) + try: waitFor emulateSocket(path) + finally: removeFile(path) diff --git a/src/nix_actor/store.nim b/src/nix_actor/store.nim index 997720d54054..5808608d1925 100644 --- a/src/nix_actor/store.nim +++ b/src/nix_actor/store.nim @@ -6,10 +6,20 @@ {.passC: "'-DSYSTEM=\"x86_64-linux\"'".} +type StdString {.importcpp: "std::string", header: "".} = object +proc data(s: StdString): pointer {.importcpp: "#.data()".} +proc len(s: StdString): csize_t {.importcpp: "#.length()".} +proc `$`*(cpp: StdString): string = + result.setLen(cpp.len) + if result.len > 0: + copyMem(addr result[0], cpp.data, result.len) + type StorePath {.importcpp: "nix::StorePath", header: "path.hh".} = object discard +var nixVersion* {.importc: "nix::nixVersion", header: "globals.hh".}: StdString + proc isDerivation*(path: StorePath): bool {.importcpp.} type diff --git a/src/nix_actor/the_protocol.nim b/src/nix_actor/the_protocol.nim deleted file mode 100644 index d7216945c960..000000000000 --- a/src/nix_actor/the_protocol.nim +++ /dev/null @@ -1,14 +0,0 @@ - -import - std/typetraits, preserves - -type - Build* {.preservesRecord: "nix-build".} = object - `input`*: string - `output`*: string - -proc `$`*(x: Build): string = - `$`(toPreserve(x)) - -proc encode*(x: Build): seq[byte] = - encode(toPreserve(x)) -- cgit 1.4.1 From 6bdd8fafad325c7507751f5821c14cb2a51e58a2 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Wed, 7 Jun 2023 18:06:11 +0100 Subject: Better build-system --- .envrc | 2 +- shell.nix | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 shell.nix diff --git a/.envrc b/.envrc index 6c4024627ec9..d324c24ca4a0 100644 --- a/.envrc +++ b/.envrc @@ -1,2 +1,2 @@ source_env .. -use flake syndicate#nix_actor +use nix diff --git a/shell.nix b/shell.nix new file mode 100644 index 000000000000..5d685670df3f --- /dev/null +++ b/shell.nix @@ -0,0 +1,4 @@ +let + flake = builtins.getFlake "syndicate"; + pkgs = import { overlays = [ flake.overlays.default ]; }; +in pkgs.nix_actor -- cgit 1.4.1 From 9a0d2a22ec9fc5bd8b7119efa447ac8dd253e72d Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sat, 10 Jun 2023 11:37:15 +0100 Subject: Translate wopQueryPathInfo, wopQueryMissing --- README.md | 40 ++- nix_actor.nimble | 2 +- protocol.prs | 3 +- src/nix_actor.nim | 22 +- src/nix_actor/protocol.nim | 2 +- src/nix_actor/sockets.nim | 675 ++++++++++++++++++++++----------------------- 6 files changed, 381 insertions(+), 363 deletions(-) diff --git a/README.md b/README.md index ac21e9414a85..f9c0b794589b 100644 --- a/README.md +++ b/README.md @@ -2,21 +2,47 @@ An actor for interacting with the [Nix](https://nixos.org/) daemon via the [Syndicated Actor Model](https://syndicate-lang.org/). +See [protocol.prs](./protocol.prs) for the Syndicate protocol [schema](https://preserves.dev/preserves-schema.html). + *This is only a proof-of-concept and is not yet useful.* ## Example configuration + +A demo script for the [Syndicate server](https://git.syndicate-lang.org/syndicate-lang/syndicate-rs), see https://synit.org/book/operation/scripting.html ``` ? $nixspace [ ? {}; in pkgs.hello" { } ?drv> [ - ? [ ] + ? [ + $log ! + ] + ] + + ? [ + $log ! + ] + + ? [ + $log ! + ] + + ? [ + $log ! ] - ? [] - ? [] + ? [ + $log ! - ? ?any [ - $log ! ] $config [ @@ -24,10 +50,12 @@ An actor for interacting with the [Nix](https://nixos.org/) daemon via the [Synd ? ?cap> [ $cap { dataspace: $nixspace + daemon-socket: "/nix/var/nix/daemon-socket/socket" + listen-socket: "/tmp/translator.worker.nix.socket" } ] ] diff --git a/nix_actor.nimble b/nix_actor.nimble index c88a14265c98..5a555ebb7be0 100644 --- a/nix_actor.nimble +++ b/nix_actor.nimble @@ -1,4 +1,4 @@ -version = "20230607" +version = "20230610" author = "Emery Hemingway" description = "Syndicated Nix Actor" license = "Unlicense" diff --git a/protocol.prs b/protocol.prs index 4e437ec1dca2..894fe35e2b0d 100644 --- a/protocol.prs +++ b/protocol.prs @@ -22,8 +22,9 @@ ActionStop = . ActionResult = . ; TODO: why not make target a singleton? -Missing = . +Missing = . +; TODO keep a few critical fields and move the rest into a dictionary PathInfo = session.buffer.len: setLen(session.buffer, wordCount) +proc send(session: Session; s: string): Future[void] = + let wordCount = 1 + ((s.len + 7) shr 3) + if session.buffer.len < wordCount: setLen(session.buffer, wordCount) session.buffer[0] = Word s.len - if wordCount > 0: - session.buffer[wordCount] = 0x00 + if s != "": + session.buffer[pred wordCount] = 0x00 copyMem(addr session.buffer[1], unsafeAddr s[0], s.len) - send(sock, addr session.buffer[0], (1 + wordCount) shl 3) + send(session.socket, addr session.buffer[0], wordCount shl 3) + +proc send(session: Session; ss: StringSeq|StringSet): Future[void] = + ## Send a set of strings. The set is sent as a contiguous buffer. + session.buffer[0] = Word ss.len + var off = 1 + for s in ss: + let + stringWordLen = (s.len + 7) shr 3 + bufferWordLen = off+1+stringWordLen + if session.buffer.len < bufferWordLen: + setLen(session.buffer, bufferWordLen) + session.buffer[off] = Word s.len + session.buffer[off+stringWordLen] = 0 # clear the aligning bits + inc(off) + copyMem(addr session.buffer[off], unsafeAddr s[0], s.len) + inc(off, stringWordLen) + send(session.socket, addr session.buffer[0], off shl 3) proc recvWord(sock: AsyncSocket): Future[Word] {.async.} = var w: Word let n = await recvInto(sock, addr w, sizeof(Word)) - if n != sizeof(Word): raise newException(ProtocolError, "short read of word") + if n != sizeof(Word): raise newException(ProtocolError, "short read") return w -proc passWord(a, b: AsyncSocket): Future[Word] {.async.} = - var w = await recvWord(a) - await send(b, addr w, sizeof(Word)) - return w +proc recvWord(session: Session): Future[Word] = + recvWord(session.socket) -proc recvString(sock: AsyncSocket): Future[string] {.async.} = - let w = await recvWord(sock) - let stringLen = int w - var s: string +proc discardWords(session: Session; n: int): Future[void] {.async.} = + if session.buffer.len < n: setLen(session.buffer, n) + let byteCount = n shl 3 + let n = await recvInto(session.socket, addr session.buffer[0], byteCount) + if n != byteCount: + raise newException(ProtocolError, "short read") + +proc recvString(socket: AsyncSocket): Future[string] {.async.} = + let stringLen = int (await recvWord(socket)) if stringLen > 0: - s.setLen((stringLen + 7) and (not 7)) - let n = await recvInto(sock, addr s[0], s.len) + var s = newString((stringLen + 7) and (not 7)) + let n = await recvInto(socket, addr s[0], s.len) if n != s.len: - raise newException(ProtocolError, "short string read") + raise newException(ProtocolError, "short read") setLen(s, stringLen) - return s + return s + return "" -proc passString(session: Session; a, b: AsyncSocket): Future[string] {.async.} = - var s = await recvString(a) - await send(session, b, s) - return s +proc recvString(session: Session): Future[string] = + recvString(session.socket) -proc passStringSeq(session: Session; a, b: AsyncSocket): Future[seq[string]] {.async.} = - let count = int(await passWord(a, b)) +proc recvStringSeq(session: Session): Future[StringSeq] {.async.} = + let count = int(await recvWord(session.socket)) var strings = newSeq[string](count) - for i in 0..= 16 - info.ultimate = (await passDaemonWord(session)) != 0 - info.sigs = await passDaemonStringSet(session) - info.ca = await passDaemonString(session) - return info - -proc passChunks(session: Session; a, b: AsyncSocket): Future[int] {.async.} = - var total: int - while true: - let chunkLen = int(await passWord(a, b)) - if chunkLen == 0: - break - else: - let wordLen = (chunkLen + 7) shr 3 - if session.buffer.len < wordLen: setLen(session.buffer, wordLen) - let recvLen = await recvInto(a, addr session.buffer[0], chunkLen) - # each chunk must be recved contiguously - if recvLen != chunkLen: - raise newException(ProtocolError, "invalid chunk read") - await send(b, addr session.buffer[0], recvLen) - inc(total, recvLen) - return total - -proc passClientChunks(session: Session): Future[int] = - passChunks(session, session.client, session.daemon) - -proc passErrorDaemonError(session: Session) {.async.} = - let - typ = await passDaemonString(session) - assert typ == "Error" - let - lvl = await passDaemonWord(session) - name = await passDaemonString(session) - msg = passDaemonString(session) - havePos = await passDaemonWord(session) - assert havePos == 0 - let - nrTraces = await passDaemonWord(session) +proc recvError(session: Session) {.async.} = + discard #[typ]# await recvString(session) + discard #[lvl]# await recvWord(session) + discard #[name]# await recvString(session) + discard #[msg]# await recvString(session) + discard #[havePos]# await recvWord(session) + let nrTraces = await recvWord(session) for i in 1..nrTraces: - let havPos = await passDaemonWord(session) - assert havPos == 0 - let msg = await passDaemonString(session) + discard #[havPos]# await recvWord(session) + discard #[msg]# await recvString(session) -proc passDaemonFields(session: Session): Future[Fields] {.async.} = - let count = await passDaemonWord(session) - var fields = newSeq[Field](count) +proc recvFields(session: Session) {.async.} = + let count = await recvWord(session) for i in 0..= 26 - await passErrorDaemonError(session) - + await recvError(session) of STDERR_NEXT: - let s = await passDaemonString(session) - + discard await recvString(session) of STDERR_START_ACTIVITY: - var act: ActionStart - act.id = BiggestInt(await passDaemonWord(session)) - act.level = BiggestInt(await passDaemonWord(session)) - act.`type` = BiggestInt(await passDaemonWord(session)) - act.text = await passDaemonString(session) - act.fields = await passDaemonFields(session) - act.parent = BiggestInt(await passDaemonWord(session)) - + discard await recvWord(session) # id + discard await recvWord(session) # level + discard await recvWord(session) # type + discard await recvString(session) # text + await recvFields(session) # fields + discard await recvWord(session) # parent of STDERR_STOP_ACTIVITY: - var act: ActionStop - act.id = BiggestInt(await passDaemonWord(session)) - + discard await recvWord(session) # id of STDERR_RESULT: var act: ActionResult - act.id = BiggestInt(await passDaemonWord(session)) - act.`type` = BiggestInt(await passDaemonWord(session)) - act.fields = await passDaemonFields(session) - + discard await recvWord(session) # id + discard await recvWord(session) # type + await recvFields(session) # fields of STDERR_LAST: break - else: raise newException(ProtocolError, "unknown work verb " & $word) -#[ -proc fromClient(miss: var Missing; socket: AsyncSocket) {.async.} = - result.targets = await passClientStringSet(session) - -proc fromDaemon(miss: var Missing; socket: AsyncSocket) {.async.} = - miss.willBuild = await passDaemonStringSet(session) - miss.willSubstitute = await passDaemonStringSet(session) - miss.unknown = await passDaemonStringSet(session) - miss.downloadSize = BiggestInt await passDaemonWord(session) - miss.narSize = BiggestInt await passDaemonWord(session) -]# - -proc loop(session: Session) {.async.} = - var chunksTotal: int - try: - while not session.client.isClosed: - let wop = await passClientWord(session) - case wop - of wopIsValidPath: - let path = await passClientString(session) - stderr.writeLine "wopIsValidPath ", path - await passWork(session) - let word = await passDaemonWord(session) - - of wopAddToStore: - assert session.version.minor >= 25 - let - name = await passClientString(session) - caMethod = await passClientString(session) - refs = await passClientStringSet(session) - repairBool = await passClientWord(session) - stderr.writeLine "wopAddToStore ", name - let n = await passClientChunks(session) - inc(chunksTotal, n) - await passWork(session) - let info = await passDaemonValidPathInfo(session, true) - - of wopAddTempRoot: - let path = await passClientString(session) - stderr.writeLine "wopAddTempRoot ", path - await passWork(session) - discard await passDaemonWord(session) - - of wopAddIndirectRoot: - let path = await passClientString(session) - stderr.writeLine "wopAddIndirectRoot ", path - await passWork(session) - discard await passDaemonWord(session) - - of wopSetOptions: - discard passClientWord(session) # keepFailed - discard passClientWord(session) # keepGoing - discard passClientWord(session) # tryFallback - discard passClientWord(session) # verbosity - discard passClientWord(session) # maxBuildJobs - discard passClientWord(session) # maxSilentTime - discard passClientWord(session) # useBuildHook - discard passClientWord(session) # verboseBuild - discard passClientWord(session) # logType - discard passClientWord(session) # printBuildTrace - discard passClientWord(session) # buildCores - discard passClientWord(session) # useSubstitutes - assert session.version.minor >= 12 - let overrides = await passClientStringMap(session) - await passWork(session) - - of wopQueryPathInfo: - assert session.version >= 17 - let path = await passClientString(session) - stderr.writeLine "wopQueryPathInfo ", path - await passWork(session) - let valid = await passDaemonWord(session) - if valid != 0: - var info = await passDaemonValidPathInfo(session, false) - info.path = path - stderr.writeLine "wopQueryPathInfo ", $info - - of wopQueryMissing: - assert session.version >= 30 - var miss: Missing - miss.targets = await passClientStringSet(session) - await passWork(session) - miss.willBuild = await passDaemonStringSet(session) - miss.willSubstitute = await passDaemonStringSet(session) - miss.unknown = await passDaemonStringSet(session) - miss.downloadSize = BiggestInt await passDaemonWord(session) - miss.narSize = BiggestInt await passDaemonWord(session) - stderr.writeLine "wopQueryMissing ", $miss - - of wopBuildPathsWithResults: - assert session.version >= 34 - let - drvs = await passClientStringSeq(session) - buildMode = await passClientWord(session) - stderr.writeLine "wopBuildPathsWithResults drvs ", $drvs - await passWork(session) - let count = await passDaemonWord(session) - for _ in 1..count: - let - path = await passDaemonString(session) - status = await passDaemonWord(session) - errorMsg = await passDaemonString(session) - timesBUild = await passDaemonWord(session) - isNonDeterministic = await passDaemonWord(session) - startTime = await passDaemonWord(session) - stopTime = await passDaemonWord(session) - outputs = await passDaemonStringMap(session) - - else: - stderr.writeLine "unknown worker op ", wop.int - break - except ProtocolError as err: - stderr.writeLine "connection terminated" - stderr.writeLine "chunk bytes transfered: ", formatSize(chunksTotal) - finally: - close(session.daemon) - close(session.client) - -proc handshake(listener: AsyncSocket): Future[Session] {.async.} = - ## Take the next connection from `listener` and return a `Session`. - let session = Session(buffer: newSeq[Word](1024)) # 8KiB - session.client = await listener.accept() - session.daemon = newAsyncSocket( - domain = AF_UNIX, - sockType = SOCK_STREAM, - protocol = cast[Protocol](0), - buffered = false) - await connectUnix(session.daemon, daemonSocketPath()) - let clientMagic = await passClientWord(session) - if clientMagic != WORKER_MAGIC_1: - raise newException(ProtocolError, "invalid protocol magic") - let daemonMagic = await passDaemonWord(session) - let daemonVersion = await passDaemonWord(session) - session.version = Version(await passClientWord(session)) - if session.version < 0x1_0a: - raise newException(ProtocolError, "obsolete protocol version") - assert session.version.minor >= 14 - discard await(passClientWord(session)) - # obsolete CPU affinity - assert session.version.minor >= 11 - discard await(passClientWord(session)) - # obsolete reserveSpace - assert session.version.minor >= 33 - let daemonVersionString = await passDaemonString(session) - assert daemonVersionString == $store.nixVersion - await passWork(session) - return session - -proc emulateSocket*(path: string) {.async, gcsafe.} = +proc daemonSocketPath: string = + getEnv( + "NIX_DAEMON_SOCKET_PATH", + "/nix/var/nix/daemon-socket/socket") + +proc newSession(socket: AsyncSocket): Session = + Session(socket: socket, buffer: newSeq[Word](512)) + +proc newSession(): Session = + newSession(newAsyncSocket( + domain = AF_UNIX, + sockType = SOCK_STREAM, + protocol = cast[Protocol](0), + buffered = false)) + +proc send(session: Session; miss: Missing) {.async.} = + await send(session, STDERR_LAST) + await send(session, miss.willBuild) + await send(session, miss.willSubstitute) + await send(session, miss.unknown) + await send(session, Word miss.downloadSize) + await send(session, Word miss.narSize) + +proc send(session: Session; info: PathInfo) {.async.} = + await send(session, STDERR_LAST) + await send(session, 1) + if info.path != "": + await send(session, info.path) + await send(session, info.deriver) + await send(session, info.narHash) + await send(session, info.references) + await send(session, Word info.registrationTime) + await send(session, Word info.narSize) + await send(session, Word info.ultimate) + await send(session, info.sigs) + await send(session, info.ca) + +proc serveClient(facet: Facet; ds: Ref; session: Session) {.async.} = + block: + let clientMagic = await recvWord(session) + if clientMagic != WORKER_MAGIC_1: + raise newException(ProtocolError, "invalid protocol magic") + await send(session, WORKER_MAGIC_2, PROTOCOL_VERSION) + let clientVersion = Version(await recvWord(session)) + if clientVersion < 0x1_21: + raise newException(ProtocolError, "obsolete protocol version") + assert clientVersion.minor >= 14 + discard await(recvWord(session)) + # obsolete CPU affinity + assert clientVersion.minor >= 11 + discard await(recvWord(session)) + # obsolete reserveSpace + assert clientVersion.minor >= 33 + await send(session, "0.0.0") + await send(session, STDERR_LAST) + while not session.socket.isClosed: + let wop = await recvWord(session.socket) + case wop + + of wopQueryPathInfo: + let + path = await recvString(session) + pat = inject(?PathInfo, { 0: ?path }) + await send(session, STDERR_NEXT) + await send(session, $pat) + run(facet) do (turn: var Turn): + onPublish(turn, ds, pat) do ( + deriver: string, + narHash: string, + references: StringSet, + registrationTime: BiggestInt, + narSize: BiggestInt, + ultimate: bool, + sigs: StringSet, + ca: string + ): + var info = PathInfo( + deriver: deriver, + narHash: narHash, + references: references, + registrationTime: registrationTime, + narSize: narSize, + ultimate: ultimate, + sigs: sigs, + ca: ca, + ) + asyncCheck(turn, send(session, info)) + + of wopQueryMissing: + var targets = toPreserve(await recvStringSeq(session)) + sort(targets.sequence) + # would prefer to use a set but that doesn't translate into a pattern + let pat = inject(?Missing, { 0: ?targets }) + # TODO send the pattern to the client as a log line + await send(session, STDERR_NEXT) + await send(session, $pat) + run(facet) do (turn: var Turn): + onPublish(turn, ds, pat) do ( + willBuild: StringSet, + willSubstitute: StringSet, + unknown: StringSet, + downloadSize: BiggestInt, + narSize: BiggestInt + ): + let miss = Missing( + willBuild: willBuild, + willSubstitute: willSubstitute, + unknown: unknown, + downloadSize: downloadSize, + narSize: narSize, + ) + asyncCheck(turn, send(session, miss)) + + of wopSetOptions: + await discardWords(session, 12) + # 01 keepFailed + # 02 keepGoing + # 03 tryFallback + # 04 verbosity + # 05 maxBuildJobs + # 06 maxSilentTime + # 07 useBuildHook + # 08 verboseBuild + # 09 logType + # 10 printBuildTrace + # 11 buildCores + # 12 useSubstitutes + let overridePairCount = await recvWord(session) + for _ in 1..overridePairCount: + discard await (recvString(session)) + discard await (recvString(session)) + await send(session, STDERR_LAST) + # all options from the client are ingored + + else: + let msg = "unhandled worker op " & $wop.int + await send(session, STDERR_NEXT) + await send(session, msg) + await send(session, STDERR_LAST) + close(session.socket) + +proc serveClientSide*(facet: Facet; ds: Ref; listener: AsyncSocket) {.async.} = + while not listener.isClosed: + let + client = await accept(listener) + fut = serveClient(facet, ds, newSession(client)) + addCallback(fut) do (): + if not client.isClosed: + close(client) + +proc bootClientSide*(facet: Facet; ds: Ref; socketPath: string) = let listener = newAsyncSocket( domain = AF_UNIX, sockType = SOCK_STREAM, protocol = cast[Protocol](0), buffered = false) - bindUnix(listener, path) + onStop(facet) do (turn: var Turn): + close(listener) + removeFile(socketPath) + removeFile(socketPath) + bindUnix(listener, socketPath) listen(listener) - stderr.writeLine "listening on ", path - while not listener.isClosed: - try: - let session = await handshake(listener) - assert not session.isNil - asyncCheck loop(session) - except ProtocolError as err: - stderr.writeLine "failed to service client, ", err.msg - -when isMainModule: - const path = "/tmp/worker.nix.socket" - if fileExists(path): removeFile(path) - try: waitFor emulateSocket(path) - finally: removeFile(path) + asyncCheck(facet, serveClientSide(facet, ds, listener)) + +proc connectDaemon(session: Session; socketPath: string) {.async.} = + await connectUnix(session.socket, socketPath) + await send(session, WORKER_MAGIC_1) + let daemonMagic = await recvWord(session) + if daemonMagic != WORKER_MAGIC_2: + raise newException(ProtocolError, "bad magic from daemon") + let daemonVersion = await recvWord(session) + session.version = min(Version daemonVersion, PROTOCOL_VERSION) + await send(session, Word session.version) + await send(session, 0) # CPU affinity + await send(session, 0) # reserve space + if session.version.minor >= 33: + discard await recvString(session) # version + if session.version.minor >= 35: + discard await recvWord(session) # remoteTrustsUs + await recvWork(session) + +proc queryMissing(session: Session; targets: StringSeq): Future[Missing] {.async.} = + var miss = Missing(targets: targets) + await send(session, wopQueryMissing) + await send(session, miss.targets) + await recvWork(session) + miss.willBuild = await recvStringSet(session) + miss.willSubstitute = await recvStringSet(session) + miss.unknown = await recvStringSet(session) + miss.downloadSize = BiggestInt await recvWord(session) + miss.narSize = BiggestInt await recvWord(session) + return miss + +proc queryPathInfo(session: Session; path: string): Future[PathInfo] {.async.} = + var info = PathInfo(path: path) + await send(session, wopQueryPathInfo) + await send(session, info.path) + await recvWork(session) + let valid = await recvWord(session) + if valid != 0: + info.deriver = await recvString(session) + info.narHash = await recvString(session) + info.references = await recvStringSet(session) + info.registrationTime = BiggestInt await recvWord(session) + info.narSize = BiggestInt await recvWord(session) + info.ultimate = (await recvWord(session)) != 0 + info.sigs = await recvStringSet(session) + info.ca = await recvString(session) + return info + +proc bootDaemonSide*(turn: var Turn; ds: Ref; socketPath: string) = + + during(turn, ds, ?Observe(pattern: !Missing) ?? {0: grab()}) do (a: Preserve[Ref]): + # cannot use `grabLit` here because an array is a compound + let + session = newSession() + fut = connectDaemon(session, socketPath) + addCallback(fut, turn) do (turn: var Turn): + read(fut) + var targets: StringSeq + doAssert targets.fromPreserve(unpackLiterals(a)) + # unpack ]> + let missFut = queryMissing(session, targets) + addCallback(missFut, turn) do (turn: var Turn): + var miss = read(missFut) + discard publish(turn, ds, miss) + do: + close(session) + + during(turn, ds, ?Observe(pattern: !PathInfo) ?? {0: grabLit()}) do (path: string): + let + session = newSession() + fut = connectDaemon(session, socketPath) + addCallback(fut, turn) do (turn: var Turn): + read(fut) + let infoFut = queryPathInfo(session, path) + addCallback(infoFut, turn) do (turn: var Turn): + var info = read(infoFut) + discard publish(turn, ds, info) + do: + close(session) -- cgit 1.4.1 From 584c01ef082566a9d3e8a7792077d5f497b3ed7d Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sun, 11 Jun 2023 12:24:25 +0100 Subject: Break up sockets module, don't link libnix, wopAddToStore --- README.md | 3 +- Tuprules.tup | 5 +- nix_actor.nimble | 3 +- protocol.prs | 67 +++++-- src/nix_actor.nim | 24 ++- src/nix_actor.nim.cfg | 1 - src/nix_actor/clients.nim | 172 ++++++++++++++++ src/nix_actor/daemons.nim | 205 +++++++++++++++++++ src/nix_actor/main.nim | 7 - src/nix_actor/protocol.nim | 74 ++++--- src/nix_actor/sockets.nim | 476 +++++++++++---------------------------------- 11 files changed, 610 insertions(+), 427 deletions(-) delete mode 100644 src/nix_actor.nim.cfg create mode 100644 src/nix_actor/clients.nim create mode 100644 src/nix_actor/daemons.nim delete mode 100644 src/nix_actor/main.nim diff --git a/README.md b/README.md index f9c0b794589b..1d07f44efbf8 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,7 @@ A demo script for the [Syndicate server](https://git.syndicate-lang.org/syndicat } }> ] - ? [ + ? [ $log ! = 1.6.10", "syndicate >= 20230530" diff --git a/protocol.prs b/protocol.prs index 894fe35e2b0d..335f54915b79 100644 --- a/protocol.prs +++ b/protocol.prs @@ -1,19 +1,19 @@ version 1 . +StringSeq = [string ...] . +StringSet = #{string} . +AttrSet = {symbol: any ...:...} . + Build = . -Realise = . +Realise = . -Instantiate = . +Instantiate = . Eval = . -Narinfo = . - -Dict = {symbol: any ...:...} . +Narinfo = . -FieldInt = int . -FieldString = string . Field = int / string . Fields = [Field ...] . @@ -22,16 +22,43 @@ ActionStop = . ActionResult = . ; TODO: why not make target a singleton? -Missing = . - -; TODO keep a few critical fields and move the rest into a dictionary -PathInfo = . +Missing = . + +; Path info for the worker protocol version 35. +LegacyPathAttrs = { + deriver: string + narHash: string + references: StringSeq ; prefer a set + registrationTime: int + narSize: int + ultimate: bool + sigs: StringSet + ca: string +} . + +AddToStoreClientAttrs = { + name: string + eris: bytes + ca-method: symbol + references: StringSeq ; prefer a set +} . + +; Intersection of the attributes needed to add a path to a store +; and the attributes returned by the daemon after adding the path. +AddToStoreAttrs = { + name: string + eris: bytes + ca-method: symbol + references: StringSeq ; prefer a set + + deriver: string + narHash: string + registrationTime: int + narSize: int + ultimate: bool + sigs: StringSet + ca: string +} . + +; Any collection of attributes describing a store path. +PathInfo = . diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 0473aa3e988d..2ad40e416d8d 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -1,18 +1,20 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -import std/[asyncdispatch, httpclient, json, osproc, parseutils, strutils, tables] +import std/[json, osproc, parseutils, strutils, tables] +import eris/memory_stores import preserves, preserves/jsonhooks import syndicate from syndicate/protocols/dataspace import Observe import ./nix_actor/protocol -import ./nix_actor/[main, sockets] +import ./nix_actor/[clients, daemons] + type Value = Preserve[void] Observe = dataspace.Observe[Ref] -proc parseArgs(args: var seq[string]; opts: Dict) = +proc parseArgs(args: var seq[string]; opts: AttrSet) = for sym, val in opts: add(args, "--" & $sym) if not val.isString "": @@ -20,7 +22,8 @@ proc parseArgs(args: var seq[string]; opts: Dict) = if fromPreserve(js, val): add(args, $js) else: stderr.writeLine "invalid option --", sym, " ", val -proc parseNarinfo(info: var Dict; text: string) = +#[ +proc parseNarinfo(info: var AttrSet; text: string) = var key, val: string off: int @@ -50,6 +53,7 @@ proc narinfo(turn: var Turn; ds: Ref; path: string) = var narinfo = Narinfo(path: path) parseNarinfo(narinfo.info, read(futBody)) discard publish(turn, ds, narinfo) +]# # I never link to openssl if I can avoid it. proc build(spec: string): Build = var execOutput = execProcess("nix", args = ["build", "--json", "--no-link", spec], options = {poUsePath}) @@ -104,8 +108,10 @@ proc bootNixFacet(turn: var Turn; ds: Ref): Facet = ass.result = eval(ass) discard publish(turn, ds, ass) + #[ during(turn, ds, ?Observe(pattern: !Narinfo) ?? {0: grabLit()}) do (path: string): narinfo(turn, ds, path) + ]# type RefArgs {.preservesDictionary.} = object @@ -115,17 +121,15 @@ type DaemonSideArgs {.preservesDictionary.} = object `daemon-socket`: string -proc bootNixActor(root: Ref; turn: var Turn) = +runActor("main") do (root: Ref; turn: var Turn): + let store = newMemoryStore() connectStdio(root, turn) during(turn, root, ?RefArgs) do (ds: Ref): discard bootNixFacet(turn, ds) during(turn, root, ?ClientSideArgs) do (socketPath: string): - bootClientSide(turn.facet, ds, socketPath) + bootClientSide(turn, ds, store, socketPath) during(turn, root, ?DaemonSideArgs) do (socketPath: string): - bootDaemonSide(turn, ds, socketPath) - -initNix() # Nix lib isn't actually being used but it's nice to know that it links. -runActor("main", bootNixActor) + bootDaemonSide(turn, ds, store, socketPath) diff --git a/src/nix_actor.nim.cfg b/src/nix_actor.nim.cfg deleted file mode 100644 index 1f92ea59c700..000000000000 --- a/src/nix_actor.nim.cfg +++ /dev/null @@ -1 +0,0 @@ -define:ssl diff --git a/src/nix_actor/clients.nim b/src/nix_actor/clients.nim new file mode 100644 index 000000000000..86c50e36a129 --- /dev/null +++ b/src/nix_actor/clients.nim @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +import std/[asyncdispatch, asyncnet, os, sets, strutils, tables] +from std/algorithm import sort + +import eris +import preserves, syndicate +import ./protocol, ./sockets + +proc sendNext(client: Session; msg: string) {.async.} = + await send(client, STDERR_NEXT) + await send(client, msg) + +proc sendWorkEnd(client: Session): Future[void] = + send(client, STDERR_LAST) + +proc send(client: Session; miss: Missing) {.async.} = + await sendWorkEnd(client) + await send(client, miss.willBuild) + await send(client, miss.willSubstitute) + await send(client, miss.unknown) + await send(client, Word miss.downloadSize) + await send(client, Word miss.narSize) + +proc send(client: Session; info: LegacyPathAttrs) {.async.} = + await send(client, info.deriver) + await send(client, info.narHash) + await send(client, info.references) + await send(client, Word info.registrationTime) + await send(client, Word info.narSize) + await send(client, Word info.ultimate) + await send(client, info.sigs) + await send(client, info.ca) + +proc sendValidInfo(client: Session; info: LegacyPathAttrs) {.async.} = + await sendWorkEnd(client) + await send(client, 1) # valid + await send(client, info) + +proc completeAddToStore(client: Session; path: string; info: LegacyPathAttrs) {.async.} = + await sendWorkEnd(client) + await send(client, path) + await send(client, info) + +proc serveClient(facet: Facet; ds: Ref; store: ErisStore; client: Session) {.async.} = + block: + let clientMagic = await recvWord(client) + if clientMagic != WORKER_MAGIC_1: + raise newException(ProtocolError, "invalid protocol magic") + await send(client, WORKER_MAGIC_2, PROTOCOL_VERSION) + let clientVersion = Version(await recvWord(client)) + if clientVersion < 0x1_21: + raise newException(ProtocolError, "obsolete protocol version") + assert clientVersion.minor >= 14 + discard await(recvWord(client)) + # obsolete CPU affinity + assert clientVersion.minor >= 11 + discard await(recvWord(client)) + # obsolete reserveSpace + assert clientVersion.minor >= 33 + await send(client, "0.0.0") + await sendWorkEnd(client) + while not client.socket.isClosed: + let wop = await recvWord(client.socket) + case wop + + of wopAddToStore: + let + name = await recvString(client) + caMethod = await recvString(client) + var storeRefs = await recvStringSeq(client) + sort(storeRefs) # sets not valid for patterns so use a sorted list + discard await recvWord(client) # repair, not implemented + let cap = await ingestChunks(client, store) + await sendNext(client, $cap & " " & name) + let attrsPat = inject(?AddToStoreAttrs, { + "name".toSymbol(Ref): ?name, + "ca-method".toSymbol(Ref): ?caMethod.toSymbol, + "references".toSymbol(Ref): ?storeRefs, + "eris".toSymbol(Ref): ?cap.bytes, + }) + # bind AddToStoreAttrs and override with some literal values + let pat = PathInfo ? { 0: grab(), 1: attrsPat } + run(facet) do (turn: var Turn): + onPublish(turn, ds, pat) do (path: string, ca: string, deriver: string, narHash: string, narSize: BiggestInt, regTime: BiggestInt, sigs: StringSet, ultimate: bool): + asyncCheck(turn, completeAddToStore(client, path, LegacyPathAttrs( + ca: ca, + deriver: deriver, + narHash: narHash, + narSize: narSize, + references: storeRefs, + registrationTime: regTime, + sigs: sigs, + ultimate: ultimate, + ))) + + of wopQueryPathInfo: + let + path = await recvString(client) + pat = PathInfo ? { 0: ?path, 1: grab() } + run(facet) do (turn: var Turn): + onPublish(turn, ds, pat) do (info: LegacyPathAttrs): + asyncCheck(turn, sendValidInfo(client, info)) + + of wopQueryMissing: + var targets = toPreserve(await recvStringSeq(client)) + sort(targets.sequence) + # would prefer to use a set but that doesn't translate into a pattern + let pat = inject(?Missing, { 0: ?targets }) + run(facet) do (turn: var Turn): + onPublish(turn, ds, pat) do ( + willBuild: StringSet, + willSubstitute: StringSet, + unknown: StringSet, + downloadSize: BiggestInt, + narSize: BiggestInt + ): + let miss = Missing( + willBuild: willBuild, + willSubstitute: willSubstitute, + unknown: unknown, + downloadSize: downloadSize, + narSize: narSize, + ) + asyncCheck(turn, send(client, miss)) + + of wopSetOptions: + await discardWords(client, 12) + # 01 keepFailed + # 02 keepGoing + # 03 tryFallback + # 04 verbosity + # 05 maxBuildJobs + # 06 maxSilentTime + # 07 useBuildHook + # 08 verboseBuild + # 09 logType + # 10 printBuildTrace + # 11 buildCores + # 12 useSubstitutes + let overridePairCount = await recvWord(client) + for _ in 1..overridePairCount: + discard await (recvString(client)) + discard await (recvString(client)) + await sendWorkEnd(client) + # all options from the client are ingored + + else: + let msg = "unhandled worker op " & $wop.int + await sendNext(client, msg) + await sendWorkEnd(client) + close(client.socket) + +proc serveClientSide(facet: Facet; ds: Ref; store: ErisStore; listener: AsyncSocket) {.async.} = + while not listener.isClosed: + let + client = await accept(listener) + fut = serveClient(facet, ds, store, newSession(client)) + addCallback(fut) do (): + if not client.isClosed: + close(client) + +proc bootClientSide*(turn: var Turn; ds: Ref; store: ErisStore; socketPath: string) = + let listener = newUnixSocket() + onStop(turn.facet) do (turn: var Turn): + close(listener) + removeFile(socketPath) + removeFile(socketPath) + bindUnix(listener, socketPath) + listen(listener) + asyncCheck(turn, serveClientSide(turn.facet, ds, store, listener)) diff --git a/src/nix_actor/daemons.nim b/src/nix_actor/daemons.nim new file mode 100644 index 000000000000..8afa7a05fe6a --- /dev/null +++ b/src/nix_actor/daemons.nim @@ -0,0 +1,205 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +import std/[asyncdispatch, asyncnet, sets, streams, strutils] +from std/algorithm import sort + +import eris +import preserves, syndicate +from syndicate/protocols/dataspace import Observe +import ./protocol, ./sockets + +type Value = Preserve[void] + +proc merge(items: varargs[Value]): Value = + # TODO: just a hack, not a proper imlementation + # https://preserves.dev/preserves.html#appendix-merging-values + result = initDictionary() + for e in items: + for (key, val) in e.pairs: + result[key] = val + cannonicalize(result) + +type Observe = dataspace.Observe[Ref] + +proc recvError(daemon: Session): Future[string] {.async.} = + discard #[typ]# await recvString(daemon) + discard #[lvl]# await recvWord(daemon) + discard #[name]# await recvString(daemon) + let msg = #[msg]# await recvString(daemon) + discard #[havePos]# await recvWord(daemon) + let nrTraces = await recvWord(daemon) + for i in 1..nrTraces: + discard #[havPos]# await recvWord(daemon) + discard #[msg]# await recvString(daemon) + return msg + +proc recvFields(daemon: Session) {.async.} = + let count = await recvWord(daemon) + for i in 0..= 33: + discard await recvString(daemon) # version + if daemon.version.minor >= 35: + discard await recvWord(daemon) # remoteTrustsUs + await recvWork(daemon) + +proc queryMissing(daemon: Session; targets: StringSeq): Future[Missing] {.async.} = + var miss = Missing(targets: targets) + await send(daemon, wopQueryMissing) + await send(daemon, miss.targets) + await recvWork(daemon) + miss.willBuild = await recvStringSet(daemon) + miss.willSubstitute = await recvStringSet(daemon) + miss.unknown = await recvStringSet(daemon) + miss.downloadSize = BiggestInt await recvWord(daemon) + miss.narSize = BiggestInt await recvWord(daemon) + return miss + +proc queryPathInfo(daemon: Session; path: string): Future[LegacyPathAttrs] {.async.} = + var info: LegacyPathAttrs + await send(daemon, wopQueryPathInfo) + await send(daemon, path) + await recvWork(daemon) + let valid = await recvWord(daemon) + if valid != 0: + info.deriver = await recvString(daemon) + info.narHash = await recvString(daemon) + info.references = await recvStringSeq(daemon) + sort(info.references) + info.registrationTime = BiggestInt await recvWord(daemon) + info.narSize = BiggestInt await recvWord(daemon) + info.ultimate = (await recvWord(daemon)) != 0 + info.sigs = await recvStringSet(daemon) + info.ca = await recvString(daemon) + return info + +proc recvLegacyPathAttrs(daemon: Session): Future[AddToStoreAttrs] {.async.} = + var info: AddToStoreAttrs + info.deriver = await recvString(daemon) + info.narHash = await recvString(daemon) + info.references = await recvStringSeq(daemon) + sort(info.references) + info.registrationTime = BiggestInt await recvWord(daemon) + info.narSize = BiggestInt await recvWord(daemon) + assert daemon.version.minor >= 16 + info.ultimate = (await recvWord(daemon)) != 0 + info.sigs = await recvStringSet(daemon) + info.ca = await recvString(daemon) + return info + +proc addToStore(daemon: Session; store: ErisStore; request: AddToStoreClientAttrs): Future[(string, AddToStoreAttrs)] {.async.} = + let + erisCap = parseCap(request.eris) + stream = newErisStream(store, erisCap) + await send(daemon, wopAddToStore) + await send(daemon, request.name) + await send(daemon, string request.`ca-method`) + await send(daemon, request.references) + await send(daemon, 0) # repair + await recoverChunks(daemon, store, erisCap) + await recvWork(daemon) + let path = await recvString(daemon) + var info = await recvLegacyPathAttrs(daemon) + info.eris = request.eris + info.`ca-method` = request.`ca-method` + info.name = request.name + info.references = request.references + return (path, info) + +proc callDaemon(turn: var Turn; path: string; action: proc (daemon: Session; turn: var Turn) {.gcsafe.}): Session = + let + daemon = newSession() + fut = connectDaemon(daemon, path) + addCallback(fut, turn) do (turn: var Turn): + read(fut) + action(daemon, turn) + return daemon + +proc bootDaemonSide*(turn: var Turn; ds: Ref; store: ErisStore; socketPath: string) = + + during(turn, ds, ?Observe(pattern: !Missing) ?? {0: grab()}) do (a: Preserve[Ref]): + # cannot use `grabLit` here because an array is a compound + # TODO: unpack to a `Pattern` + let daemon = callDaemon(turn, socketPath) do (daemon: Session; turn: var Turn): + var targets: StringSeq + doAssert targets.fromPreserve(unpackLiterals(a)) + # unpack ]> + let missFut = queryMissing(daemon, targets) + addCallback(missFut, turn) do (turn: var Turn): + close(daemon) + var miss = read(missFut) + discard publish(turn, ds, miss) + do: + close(daemon) + + during(turn, ds, ?Observe(pattern: !PathInfo) ?? {0: grabLit()}) do (path: string): + let daemon = callDaemon(turn, socketPath) do (daemon: Session; turn: var Turn): + let infoFut = queryPathInfo(daemon, path) + addCallback(infoFut, turn) do (turn: var Turn): + close(daemon) + var info = read(infoFut) + discard publish(turn, ds, initRecord("path", path.toPreserve, info.toPreserve)) + do: + close(daemon) + + during(turn, ds, ?Observe(pattern: !PathInfo) ?? {1: grabDict()}) do (pat: Value): + var daemon: Session + var request: AddToStoreClientAttrs + if request.fromPreserve(unpackLiterals pat): + daemon = callDaemon(turn, socketPath) do (daemon: Session; turn: var Turn): + let fut = addToStore(daemon, store, request) + addCallback(fut, turn) do (turn: var Turn): + close(daemon) + var (path, info) = read(fut) + discard publish(turn, ds, initRecord("path", path.toPreserve, info.toPreserve)) + do: + close(daemon) diff --git a/src/nix_actor/main.nim b/src/nix_actor/main.nim deleted file mode 100644 index 072e97e589e5..000000000000 --- a/src/nix_actor/main.nim +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-FileCopyrightText: ☭ Emery Hemingway -# SPDX-License-Identifier: Unlicense - -{.passC: staticExec("pkg-config --cflags nix-main").} -{.passL: staticExec("pkg-config --libs nix-main").} - -proc initNix*() {.importcpp: "nix::initNix", header: "shared.hh".} diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 1b38082bf26d..6f316278dd27 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -8,21 +8,32 @@ type `options`*: Table[Symbol, Preserve[void]] `result`*: Preserve[void] + AttrSet* = Table[Symbol, Preserve[void]] Realise* {.preservesRecord: "realise".} = object `drv`*: string - `outputs`*: seq[string] + `outputs`*: StringSeq + + LegacyPathAttrs* {.preservesDictionary.} = object + `ca`*: string + `deriver`*: string + `narHash`*: string + `narSize`*: BiggestInt + `references`*: StringSeq + `registrationTime`*: BiggestInt + `sigs`*: StringSet + `ultimate`*: bool Missing* {.preservesRecord: "missing".} = object - `targets`*: seq[string] - `willBuild`*: HashSet[string] - `willSubstitute`*: HashSet[string] - `unknown`*: HashSet[string] + `targets`*: StringSeq + `willBuild`*: StringSet + `willSubstitute`*: StringSet + `unknown`*: StringSet `downloadSize`*: BiggestInt `narSize`*: BiggestInt Narinfo* {.preservesRecord: "narinfo".} = object `path`*: string - `info`*: Dict + `info`*: AttrSet FieldKind* {.pure.} = enum `int`, `string` @@ -35,18 +46,30 @@ type `string`*: string - PathInfo* {.preservesRecord: "path-info".} = object - `path`*: string + StringSet* = HashSet[string] + AddToStoreAttrs* {.preservesDictionary.} = object + `ca`*: string + `ca-method`*: Symbol `deriver`*: string + `eris`*: seq[byte] + `name`*: string `narHash`*: string - `references`*: HashSet[string] - `registrationTime`*: BiggestInt `narSize`*: BiggestInt + `references`*: StringSeq + `registrationTime`*: BiggestInt + `sigs`*: StringSet `ultimate`*: bool - `sigs`*: HashSet[string] - `ca`*: string - Dict* = Table[Symbol, Preserve[void]] + AddToStoreClientAttrs* {.preservesDictionary.} = object + `ca-method`*: Symbol + `eris`*: seq[byte] + `name`*: string + `references`*: StringSeq + + PathInfo* {.preservesRecord: "path".} = object + `path`*: string + `attrs`*: AttrSet + Build* {.preservesRecord: "nix-build".} = object `input`*: string `output`*: Preserve[void] @@ -60,13 +83,12 @@ type `fields`*: Fields `parent`*: BiggestInt - FieldString* = string Instantiate* {.preservesRecord: "instantiate".} = object `expr`*: string - `options`*: Dict + `options`*: AttrSet `result`*: Preserve[void] - FieldInt* = BiggestInt + StringSeq* = seq[string] ActionStop* {.preservesRecord: "stop".} = object `id`*: BiggestInt @@ -75,24 +97,32 @@ type `type`*: BiggestInt `fields`*: Fields -proc `$`*(x: Eval | Realise | Missing | Narinfo | Field | PathInfo | Dict | +proc `$`*(x: Eval | AttrSet | Realise | LegacyPathAttrs | Missing | Narinfo | + Field | + StringSet | + AddToStoreAttrs | + AddToStoreClientAttrs | + PathInfo | Build | Fields | ActionStart | - FieldString | Instantiate | - FieldInt | + StringSeq | ActionStop | ActionResult): string = `$`(toPreserve(x)) -proc encode*(x: Eval | Realise | Missing | Narinfo | Field | PathInfo | Dict | +proc encode*(x: Eval | AttrSet | Realise | LegacyPathAttrs | Missing | Narinfo | + Field | + StringSet | + AddToStoreAttrs | + AddToStoreClientAttrs | + PathInfo | Build | Fields | ActionStart | - FieldString | Instantiate | - FieldInt | + StringSeq | ActionStop | ActionResult): seq[byte] = encode(toPreserve(x)) diff --git a/src/nix_actor/sockets.nim b/src/nix_actor/sockets.nim index b333c040863c..2bc02b5a74cf 100644 --- a/src/nix_actor/sockets.nim +++ b/src/nix_actor/sockets.nim @@ -1,93 +1,96 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -import std/[algorithm, asyncdispatch, asyncnet, os, sets, strtabs, strutils] -from std/nativesockets import AF_INET, AF_UNIX, SOCK_STREAM, Protocol +## Common module for communicating with Nix clients and daemons. +import std/[asyncdispatch, asyncnet, sets, strtabs, strutils, tables] +from std/nativesockets import AF_UNIX, SOCK_STREAM, Protocol + +import eris import preserves, syndicate -from syndicate/protocols/dataspace import Observe + import ./protocol {.pragma: workerProtocol, importc, header: "worker-protocol.hh".} -type Word = uint64 -proc `$`(w: Word): string = toHex(w) +type Word* = uint64 -const - WORKER_MAGIC_1 = 0x6E697863 - WORKER_MAGIC_2 = 0x6478696F - PROTOCOL_VERSION = 0x100 or 35 +proc `[]=`*[T](attrs: var AttrSet; key: string; val: T) = + attrs[Symbol key] = val.toPreserve - STDERR_NEXT = 0x6F6C6d67 - STDERR_READ = 0x64617461 - STDERR_WRITE = 0x64617416 - STDERR_LAST = 0x616C7473 - STDERR_ERROR = 0x63787470 - STDERR_START_ACTIVITY = 0x53545254 - STDERR_STOP_ACTIVITY = 0x53544F50 - STDERR_RESULT = 0x52534C54 - - wopIsValidPath = 1 - wopHasSubstitutes = 3 - wopQueryReferrers = 6 - wopAddToStore = 7 - wopBuildPaths = 9 - wopEnsurePath = 10 - wopAddTempRoot = 11 - wopAddIndirectRoot = 12 - wopSyncWithGC = 13 - wopFindRoots = 14 - wopSetOptions = 19 - wopCollectGarbage = 20 - wopQuerySubstitutablePathInfo = 21 - wopQueryAllValidPaths = 23 - wopQueryFailedPaths = 24 - wopClearFailedPaths = 25 - wopQueryPathInfo = 26 - wopQueryPathFromHashPart = 29 - wopQuerySubstitutablePathInfos = 30 - wopQueryValidPaths = 31 - wopQuerySubstitutablePaths = 32 - wopQueryValidDerivers = 33 - wopOptimiseStore = 34 - wopVerifyStore = 35 - wopBuildDerivation = 36 - wopAddSignatures = 37 - wopNarFromPath = 38 - wopAddToStoreNar = 39 - wopQueryMissing = 40 - wopQueryDerivationOutputMap = 41 - wopRegisterDrvOutput = 42 - wopQueryRealisation = 43 - wopAddMultipleToStore = 44 - wopAddBuildLog = 45 - wopBuildPathsWithResults = 46 +const + WORKER_MAGIC_1* = 0x6E697863 + WORKER_MAGIC_2* = 0x6478696F + PROTOCOL_VERSION* = 0x100 or 35 + + STDERR_NEXT* = 0x6F6C6d67 + STDERR_READ* = 0x64617461 + STDERR_WRITE* = 0x64617416 + STDERR_LAST* = 0x616C7473 + STDERR_ERROR* = 0x63787470 + STDERR_START_ACTIVITY* = 0x53545254 + STDERR_STOP_ACTIVITY* = 0x53544F50 + STDERR_RESULT* = 0x52534C54 + + wopIsValidPath* = 1 + wopHasSubstitutes* = 3 + wopQueryReferrers* = 6 + wopAddToStore* = 7 + wopBuildPaths* = 9 + wopEnsurePath* = 10 + wopAddTempRoot* = 11 + wopAddIndirectRoot* = 12 + wopSyncWithGC* = 13 + wopFindRoots* = 14 + wopSetOptions* = 19 + wopCollectGarbage* = 20 + wopQuerySubstitutablePathInfo* = 21 + wopQueryAllValidPaths* = 23 + wopQueryFailedPaths* = 24 + wopClearFailedPaths* = 25 + wopQueryPathInfo* = 26 + wopQueryPathFromHashPart* = 29 + wopQuerySubstitutablePathInfos* = 30 + wopQueryValidPaths* = 31 + wopQuerySubstitutablePaths* = 32 + wopQueryValidDerivers* = 33 + wopOptimiseStore* = 34 + wopVerifyStore* = 35 + wopBuildDerivation* = 36 + wopAddSignatures* = 37 + wopNarFromPath* = 38 + wopAddToStoreNar* = 39 + wopQueryMissing* = 40 + wopQueryDerivationOutputMap* = 41 + wopRegisterDrvOutput* = 42 + wopQueryRealisation* = 43 + wopAddMultipleToStore* = 44 + wopAddBuildLog* = 45 + wopBuildPathsWithResults* = 46 type - ProtocolError = object of IOError - Version = uint16 - StringSeq = seq[string] - StringSet = HashSet[string] - Session = ref object - socket: AsyncSocket - buffer: seq[Word] - version: Version - Observe = dataspace.Observe[Ref] + ProtocolError* = object of IOError + Version* = uint16 -func major(version: Version): uint16 = version and 0xff00 -func minor(version: Version): uint16 = version and 0x00ff + Session* = ref object + socket*: AsyncSocket + buffer*: seq[Word] + version*: Version -proc close(session: Session) = +func major*(version: Version): uint16 = version and 0xff00 +func minor*(version: Version): uint16 = version and 0x00ff + +proc close*(session: Session) = close(session.socket) reset(session.buffer) -proc send(session: Session; words: varargs[Word]): Future[void] = +proc send*(session: Session; words: varargs[Word]): Future[void] = if session.buffer.len < words.len: session.buffer.setLen(words.len) for i, word in words: session.buffer[i] = word send(session.socket, addr session.buffer[0], words.len shl 3) -proc send(session: Session; s: string): Future[void] = +proc send*(session: Session; s: string): Future[void] = let wordCount = 1 + ((s.len + 7) shr 3) if session.buffer.len < wordCount: setLen(session.buffer, wordCount) session.buffer[0] = Word s.len @@ -96,7 +99,7 @@ proc send(session: Session; s: string): Future[void] = copyMem(addr session.buffer[1], unsafeAddr s[0], s.len) send(session.socket, addr session.buffer[0], wordCount shl 3) -proc send(session: Session; ss: StringSeq|StringSet): Future[void] = +proc send*(session: Session; ss: StringSeq|StringSet): Future[void] = ## Send a set of strings. The set is sent as a contiguous buffer. session.buffer[0] = Word ss.len var off = 1 @@ -113,23 +116,23 @@ proc send(session: Session; ss: StringSeq|StringSet): Future[void] = inc(off, stringWordLen) send(session.socket, addr session.buffer[0], off shl 3) -proc recvWord(sock: AsyncSocket): Future[Word] {.async.} = +proc recvWord*(sock: AsyncSocket): Future[Word] {.async.} = var w: Word let n = await recvInto(sock, addr w, sizeof(Word)) if n != sizeof(Word): raise newException(ProtocolError, "short read") return w -proc recvWord(session: Session): Future[Word] = +proc recvWord*(session: Session): Future[Word] = recvWord(session.socket) -proc discardWords(session: Session; n: int): Future[void] {.async.} = +proc discardWords*(session: Session; n: int): Future[void] {.async.} = if session.buffer.len < n: setLen(session.buffer, n) let byteCount = n shl 3 let n = await recvInto(session.socket, addr session.buffer[0], byteCount) if n != byteCount: raise newException(ProtocolError, "short read") -proc recvString(socket: AsyncSocket): Future[string] {.async.} = +proc recvString*(socket: AsyncSocket): Future[string] {.async.} = let stringLen = int (await recvWord(socket)) if stringLen > 0: var s = newString((stringLen + 7) and (not 7)) @@ -140,310 +143,61 @@ proc recvString(socket: AsyncSocket): Future[string] {.async.} = return s return "" -proc recvString(session: Session): Future[string] = +proc recvString*(session: Session): Future[string] = recvString(session.socket) -proc recvStringSeq(session: Session): Future[StringSeq] {.async.} = +proc recvStringSeq*(session: Session): Future[StringSeq] {.async.} = let count = int(await recvWord(session.socket)) var strings = newSeq[string](count) for i in 0..= 14 - discard await(recvWord(session)) - # obsolete CPU affinity - assert clientVersion.minor >= 11 - discard await(recvWord(session)) - # obsolete reserveSpace - assert clientVersion.minor >= 33 - await send(session, "0.0.0") - await send(session, STDERR_LAST) - while not session.socket.isClosed: - let wop = await recvWord(session.socket) - case wop - - of wopQueryPathInfo: - let - path = await recvString(session) - pat = inject(?PathInfo, { 0: ?path }) - await send(session, STDERR_NEXT) - await send(session, $pat) - run(facet) do (turn: var Turn): - onPublish(turn, ds, pat) do ( - deriver: string, - narHash: string, - references: StringSet, - registrationTime: BiggestInt, - narSize: BiggestInt, - ultimate: bool, - sigs: StringSet, - ca: string - ): - var info = PathInfo( - deriver: deriver, - narHash: narHash, - references: references, - registrationTime: registrationTime, - narSize: narSize, - ultimate: ultimate, - sigs: sigs, - ca: ca, - ) - asyncCheck(turn, send(session, info)) - - of wopQueryMissing: - var targets = toPreserve(await recvStringSeq(session)) - sort(targets.sequence) - # would prefer to use a set but that doesn't translate into a pattern - let pat = inject(?Missing, { 0: ?targets }) - # TODO send the pattern to the client as a log line - await send(session, STDERR_NEXT) - await send(session, $pat) - run(facet) do (turn: var Turn): - onPublish(turn, ds, pat) do ( - willBuild: StringSet, - willSubstitute: StringSet, - unknown: StringSet, - downloadSize: BiggestInt, - narSize: BiggestInt - ): - let miss = Missing( - willBuild: willBuild, - willSubstitute: willSubstitute, - unknown: unknown, - downloadSize: downloadSize, - narSize: narSize, - ) - asyncCheck(turn, send(session, miss)) - - of wopSetOptions: - await discardWords(session, 12) - # 01 keepFailed - # 02 keepGoing - # 03 tryFallback - # 04 verbosity - # 05 maxBuildJobs - # 06 maxSilentTime - # 07 useBuildHook - # 08 verboseBuild - # 09 logType - # 10 printBuildTrace - # 11 buildCores - # 12 useSubstitutes - let overridePairCount = await recvWord(session) - for _ in 1..overridePairCount: - discard await (recvString(session)) - discard await (recvString(session)) - await send(session, STDERR_LAST) - # all options from the client are ingored - - else: - let msg = "unhandled worker op " & $wop.int - await send(session, STDERR_NEXT) - await send(session, msg) - await send(session, STDERR_LAST) - close(session.socket) - -proc serveClientSide*(facet: Facet; ds: Ref; listener: AsyncSocket) {.async.} = - while not listener.isClosed: - let - client = await accept(listener) - fut = serveClient(facet, ds, newSession(client)) - addCallback(fut) do (): - if not client.isClosed: - close(client) - -proc bootClientSide*(facet: Facet; ds: Ref; socketPath: string) = - let listener = newAsyncSocket( +proc newUnixSocket*(): AsyncSocket = + newAsyncSocket( domain = AF_UNIX, sockType = SOCK_STREAM, protocol = cast[Protocol](0), - buffered = false) - onStop(facet) do (turn: var Turn): - close(listener) - removeFile(socketPath) - removeFile(socketPath) - bindUnix(listener, socketPath) - listen(listener) - asyncCheck(facet, serveClientSide(facet, ds, listener)) - -proc connectDaemon(session: Session; socketPath: string) {.async.} = - await connectUnix(session.socket, socketPath) - await send(session, WORKER_MAGIC_1) - let daemonMagic = await recvWord(session) - if daemonMagic != WORKER_MAGIC_2: - raise newException(ProtocolError, "bad magic from daemon") - let daemonVersion = await recvWord(session) - session.version = min(Version daemonVersion, PROTOCOL_VERSION) - await send(session, Word session.version) - await send(session, 0) # CPU affinity - await send(session, 0) # reserve space - if session.version.minor >= 33: - discard await recvString(session) # version - if session.version.minor >= 35: - discard await recvWord(session) # remoteTrustsUs - await recvWork(session) + buffered = false, + ) -proc queryMissing(session: Session; targets: StringSeq): Future[Missing] {.async.} = - var miss = Missing(targets: targets) - await send(session, wopQueryMissing) - await send(session, miss.targets) - await recvWork(session) - miss.willBuild = await recvStringSet(session) - miss.willSubstitute = await recvStringSet(session) - miss.unknown = await recvStringSet(session) - miss.downloadSize = BiggestInt await recvWord(session) - miss.narSize = BiggestInt await recvWord(session) - return miss - -proc queryPathInfo(session: Session; path: string): Future[PathInfo] {.async.} = - var info = PathInfo(path: path) - await send(session, wopQueryPathInfo) - await send(session, info.path) - await recvWork(session) - let valid = await recvWord(session) - if valid != 0: - info.deriver = await recvString(session) - info.narHash = await recvString(session) - info.references = await recvStringSet(session) - info.registrationTime = BiggestInt await recvWord(session) - info.narSize = BiggestInt await recvWord(session) - info.ultimate = (await recvWord(session)) != 0 - info.sigs = await recvStringSet(session) - info.ca = await recvString(session) - return info +proc newSession*(socket: AsyncSocket): Session = + Session(socket: socket, buffer: newSeq[Word](512)) -proc bootDaemonSide*(turn: var Turn; ds: Ref; socketPath: string) = +proc newSession*(): Session = + newUnixSocket().newSession() - during(turn, ds, ?Observe(pattern: !Missing) ?? {0: grab()}) do (a: Preserve[Ref]): - # cannot use `grabLit` here because an array is a compound - let - session = newSession() - fut = connectDaemon(session, socketPath) - addCallback(fut, turn) do (turn: var Turn): - read(fut) - var targets: StringSeq - doAssert targets.fromPreserve(unpackLiterals(a)) - # unpack ]> - let missFut = queryMissing(session, targets) - addCallback(missFut, turn) do (turn: var Turn): - var miss = read(missFut) - discard publish(turn, ds, miss) - do: - close(session) - - during(turn, ds, ?Observe(pattern: !PathInfo) ?? {0: grabLit()}) do (path: string): - let - session = newSession() - fut = connectDaemon(session, socketPath) - addCallback(fut, turn) do (turn: var Turn): - read(fut) - let infoFut = queryPathInfo(session, path) - addCallback(infoFut, turn) do (turn: var Turn): - var info = read(infoFut) - discard publish(turn, ds, info) - do: - close(session) +proc ingestChunks*(session: Session; store: ErisStore): Future[ErisCap] {.async.} = + var ingest: ErisIngest + while true: + let chunkLen = int await recvWord(session) + if ingest.isNil: + ingest = newErisIngest( + store, recommendedChunkSize(chunkLen), convergentMode) + if chunkLen == 0: + break + else: + let wordLen = (chunkLen + 7) shr 3 + if session.buffer.len < wordLen: setLen(session.buffer, wordLen) + let recvLen = await recvInto(session.socket, addr session.buffer[0], chunkLen) + # each chunk must be received contiguously + if recvLen != chunkLen: + raise newException(ProtocolError, "invalid chunk read") + await append(ingest, addr session.buffer[0], chunkLen) + var cap = await cap(ingest) + return cap + +proc recoverChunks*(session: Session; store: ErisStore; cap: ErisCap) {.async.} = + let stream = newErisStream(store, cap) + session.buffer.setLen(succ(cap.chunkSize.int shr 3)) + while true: + let n = await stream.readBuffer(addr session.buffer[1], cap.chunkSize.int) + session.buffer[0] = Word n + await send(session.socket, addr session.buffer[0], 8+n) + if n == 0: break + close(stream) -- cgit 1.4.1 From a6aeaadf8102382d165eaa9a6e1064acc48f47cd Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 12 Jun 2023 23:42:24 +0100 Subject: Make wop* an enum --- nix_actor.nimble | 2 +- src/nix_actor/clients.nim | 6 ++-- src/nix_actor/daemons.nim | 6 ++-- src/nix_actor/sockets.nim | 79 ++++++++++++++++++++++++++--------------------- 4 files changed, 52 insertions(+), 41 deletions(-) diff --git a/nix_actor.nimble b/nix_actor.nimble index 2a2789540dc2..6d393f2858d9 100644 --- a/nix_actor.nimble +++ b/nix_actor.nimble @@ -1,4 +1,4 @@ -version = "20230611" +version = "20230612" author = "Emery Hemingway" description = "Syndicated Nix Actor" license = "Unlicense" diff --git a/src/nix_actor/clients.nim b/src/nix_actor/clients.nim index 86c50e36a129..0ee95eb621a4 100644 --- a/src/nix_actor/clients.nim +++ b/src/nix_actor/clients.nim @@ -62,7 +62,9 @@ proc serveClient(facet: Facet; ds: Ref; store: ErisStore; client: Session) {.asy await send(client, "0.0.0") await sendWorkEnd(client) while not client.socket.isClosed: - let wop = await recvWord(client.socket) + let + w = await recvWord(client.socket) + wop = WorkerOperation(w) case wop of wopAddToStore: @@ -147,7 +149,7 @@ proc serveClient(facet: Facet; ds: Ref; store: ErisStore; client: Session) {.asy # all options from the client are ingored else: - let msg = "unhandled worker op " & $wop.int + let msg = "unhandled worker op " & $wop await sendNext(client, msg) await sendWorkEnd(client) close(client.socket) diff --git a/src/nix_actor/daemons.nim b/src/nix_actor/daemons.nim index 8afa7a05fe6a..8ea436121cb8 100644 --- a/src/nix_actor/daemons.nim +++ b/src/nix_actor/daemons.nim @@ -94,7 +94,7 @@ proc connectDaemon(daemon: Session; socketPath: string) {.async.} = proc queryMissing(daemon: Session; targets: StringSeq): Future[Missing] {.async.} = var miss = Missing(targets: targets) - await send(daemon, wopQueryMissing) + await send(daemon, Word wopQueryMissing) await send(daemon, miss.targets) await recvWork(daemon) miss.willBuild = await recvStringSet(daemon) @@ -106,7 +106,7 @@ proc queryMissing(daemon: Session; targets: StringSeq): Future[Missing] {.async. proc queryPathInfo(daemon: Session; path: string): Future[LegacyPathAttrs] {.async.} = var info: LegacyPathAttrs - await send(daemon, wopQueryPathInfo) + await send(daemon, Word wopQueryPathInfo) await send(daemon, path) await recvWork(daemon) let valid = await recvWord(daemon) @@ -140,7 +140,7 @@ proc addToStore(daemon: Session; store: ErisStore; request: AddToStoreClientAttr let erisCap = parseCap(request.eris) stream = newErisStream(store, erisCap) - await send(daemon, wopAddToStore) + await send(daemon, Word wopAddToStore) await send(daemon, request.name) await send(daemon, string request.`ca-method`) await send(daemon, request.references) diff --git a/src/nix_actor/sockets.nim b/src/nix_actor/sockets.nim index 2bc02b5a74cf..3d67e7a1859b 100644 --- a/src/nix_actor/sockets.nim +++ b/src/nix_actor/sockets.nim @@ -32,41 +32,50 @@ const STDERR_STOP_ACTIVITY* = 0x53544F50 STDERR_RESULT* = 0x52534C54 - wopIsValidPath* = 1 - wopHasSubstitutes* = 3 - wopQueryReferrers* = 6 - wopAddToStore* = 7 - wopBuildPaths* = 9 - wopEnsurePath* = 10 - wopAddTempRoot* = 11 - wopAddIndirectRoot* = 12 - wopSyncWithGC* = 13 - wopFindRoots* = 14 - wopSetOptions* = 19 - wopCollectGarbage* = 20 - wopQuerySubstitutablePathInfo* = 21 - wopQueryAllValidPaths* = 23 - wopQueryFailedPaths* = 24 - wopClearFailedPaths* = 25 - wopQueryPathInfo* = 26 - wopQueryPathFromHashPart* = 29 - wopQuerySubstitutablePathInfos* = 30 - wopQueryValidPaths* = 31 - wopQuerySubstitutablePaths* = 32 - wopQueryValidDerivers* = 33 - wopOptimiseStore* = 34 - wopVerifyStore* = 35 - wopBuildDerivation* = 36 - wopAddSignatures* = 37 - wopNarFromPath* = 38 - wopAddToStoreNar* = 39 - wopQueryMissing* = 40 - wopQueryDerivationOutputMap* = 41 - wopRegisterDrvOutput* = 42 - wopQueryRealisation* = 43 - wopAddMultipleToStore* = 44 - wopAddBuildLog* = 45 - wopBuildPathsWithResults* = 46 +type WorkerOperation* = enum + wopIsValidPath = 1, + wopHasSubstitutes = 3, + wopQueryPathHash = 4, # obsolete + wopQueryReferences = 5, # obsolete + wopQueryReferrers = 6, + wopAddToStore = 7, + wopAddTextToStore = 8, # obsolete since 1.25, Nix 3.0. Use wopAddToStore + wopBuildPaths = 9, + wopEnsurePath = 10, + wopAddTempRoot = 11, + wopAddIndirectRoot = 12, + wopSyncWithGC = 13, + wopFindRoots = 14, + wopExportPath = 16, # obsolete + wopQueryDeriver = 18, # obsolete + wopSetOptions = 19, + wopCollectGarbage = 20, + wopQuerySubstitutablePathInfo = 21, + wopQueryDerivationOutputs = 22, # obsolete + wopQueryAllValidPaths = 23, + wopQueryFailedPaths = 24, + wopClearFailedPaths = 25, + wopQueryPathInfo = 26, + wopImportPaths = 27, # obsolete + wopQueryDerivationOutputNames = 28, # obsolete + wopQueryPathFromHashPart = 29, + wopQuerySubstitutablePathInfos = 30, + wopQueryValidPaths = 31, + wopQuerySubstitutablePaths = 32, + wopQueryValidDerivers = 33, + wopOptimiseStore = 34, + wopVerifyStore = 35, + wopBuildDerivation = 36, + wopAddSignatures = 37, + wopNarFromPath = 38, + wopAddToStoreNar = 39, + wopQueryMissing = 40, + wopQueryDerivationOutputMap = 41, + wopRegisterDrvOutput = 42, + wopQueryRealisation = 43, + wopAddMultipleToStore = 44, + wopAddBuildLog = 45, + wopBuildPathsWithResults = 46, type ProtocolError* = object of IOError -- cgit 1.4.1 From d51d9b3c34a887587d17985add18a109001f5d16 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 13 Jun 2023 17:46:01 +0100 Subject: Native evaluation (rather than by subprocess) --- nix_actor.nimble | 2 +- src/Tupfile | 4 +- src/nix_actor.nim | 80 ++++++++++++++++++++-------- src/nix_actor.nim.cfg | 1 + src/nix_actor/daemons.nim | 21 ++------ src/nix_actor/libnix/libexpr.nim | 104 +++++++++++++++++++++++++++++++++++++ src/nix_actor/libnix/main.nim | 7 +++ src/nix_actor/libnix/seepuspus.hh | 21 ++++++++ src/nix_actor/libnix/stdpuspus.nim | 24 +++++++++ src/nix_actor/libnix/store.nim | 31 +++++++++++ src/nix_actor/sockets.nim | 1 + src/nix_actor/store.nim | 31 ----------- 12 files changed, 254 insertions(+), 73 deletions(-) create mode 100644 src/nix_actor.nim.cfg create mode 100644 src/nix_actor/libnix/libexpr.nim create mode 100644 src/nix_actor/libnix/main.nim create mode 100644 src/nix_actor/libnix/seepuspus.hh create mode 100644 src/nix_actor/libnix/stdpuspus.nim create mode 100644 src/nix_actor/libnix/store.nim delete mode 100644 src/nix_actor/store.nim diff --git a/nix_actor.nimble b/nix_actor.nimble index 6d393f2858d9..0ab4c42947c1 100644 --- a/nix_actor.nimble +++ b/nix_actor.nimble @@ -1,4 +1,4 @@ -version = "20230612" +version = "20230613" author = "Emery Hemingway" description = "Syndicated Nix Actor" license = "Unlicense" diff --git a/src/Tupfile b/src/Tupfile index 2036a765b33e..7bd661c4a4c8 100644 --- a/src/Tupfile +++ b/src/Tupfile @@ -1,3 +1,3 @@ include_rules -: nix_actor.nim | $(SYNDICATE_PROTOCOL) ./ |> !nim_bin |> {bin} -: {bin} |> !assert_built |> +: foreach *.nim | $(SYNDICATE_PROTOCOL) ./ |> !nim_bin |> {bin} +: foreach {bin} |> !assert_built |> diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 2ad40e416d8d..ce547758664a 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -1,19 +1,57 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -import std/[json, osproc, parseutils, strutils, tables] +import std/[json, os, osproc, parseutils, strutils, tables] import eris/memory_stores import preserves, preserves/jsonhooks import syndicate from syndicate/protocols/dataspace import Observe -import ./nix_actor/protocol import ./nix_actor/[clients, daemons] - +import ./nix_actor/libnix/[libexpr, main, store] +import ./nix_actor/protocol type Value = Preserve[void] Observe = dataspace.Observe[Ref] +proc toPreserve(val: libexpr.ValueObj | libExpr.ValuePtr; state: EvalState; E = void): Preserve[E] {.gcsafe.} = + ## Convert a Nix value to a Preserves value. + case val.kind + of nThunk: + result = initRecord[E]("thunk") + of nInt: + result = val.integer.toPreserve(E) + of nFloat: + result = val.fpoint.toPreserve(E) + of nBool: + result = val.boolean.toPreserve(E) + of nString: + result = val.shallowString.toPreserve(E) + of nPath: + result = toSymbol($val.path, E) + of nNull: + result = initRecord[E]("null") + of nAttrs: + result = initDictionary(E) + for key, val in val.pairs: + result[symbolString(state, key).toSymbol(E)] = val.toPreserve(state, E) + of nList: + result = initSequence(0, E) + for e in val.items: + result.sequence.add(e.toPreserve(state, E)) + of nFunction: + result = initRecord[E]("func") + of nExternal: + result = initRecord[E]("external") + +proc eval(state: EvalState; code: string): Value = + ## Evaluate Nix `code` to a Preserves value. + var nixVal: libexpr.ValueObj + let expr = state.parseExprFromString(code, getCurrentDir()) + state.eval(expr, nixVal) + state.forceValueDeep(nixVal) + nixVal.toPreserve(state, void) + proc parseArgs(args: var seq[string]; opts: AttrSet) = for sym, val in opts: add(args, "--" & $sym) @@ -71,15 +109,6 @@ proc instantiate(instantiate: Instantiate): Value = var execOutput = strip execProcess(cmd, args = args, options = {poUsePath}) execOutput.toPreserve -proc eval(eval: Eval): Value = - const cmd = "nix" - var args = @["eval", "--expr", eval.expr] - parseArgs(args, eval.options) - var execOutput = strip execProcess(cmd, args = args, options = {poUsePath}) - if execOutput != "": - var js = parseJson(execOutput) - result = js.toPreserve - proc bootNixFacet(turn: var Turn; ds: Ref): Facet = # let store = openStore() result = inFacet(turn) do (turn: var Turn): @@ -100,14 +129,6 @@ proc bootNixFacet(turn: var Turn; ds: Ref): Facet = ass.result = instantiate(ass) discard publish(turn, ds, ass) - during(turn, ds, ?Observe(pattern: !Eval) ?? {0: grabLit(), 1: grabDict()}) do (e: string, o: Value): - var ass = Eval(expr: e) - if not fromPreserve(ass.options, unpackLiterals(o)): - stderr.writeLine "invalid options ", o - else: - ass.result = eval(ass) - discard publish(turn, ds, ass) - #[ during(turn, ds, ?Observe(pattern: !Narinfo) ?? {0: grabLit()}) do (path: string): narinfo(turn, ds, path) @@ -121,15 +142,28 @@ type DaemonSideArgs {.preservesDictionary.} = object `daemon-socket`: string +main.initNix() +libexpr.initGC() + runActor("main") do (root: Ref; turn: var Turn): - let store = newMemoryStore() + let + erisStore = newMemoryStore() + nixStore = openStore() + nixState = newEvalState(nixStore) connectStdio(root, turn) during(turn, root, ?RefArgs) do (ds: Ref): discard bootNixFacet(turn, ds) + during(turn, ds, ?Observe(pattern: !Eval) ?? {0: grabLit(), 1: grabDict()}) do (e: string, o: Assertion): + var ass = Eval(expr: e) + doAssert fromPreserve(ass.options, unpackLiterals(o)) + # unused options + ass.result = eval(nixState, ass.expr) + discard publish(turn, ds, ass) + during(turn, root, ?ClientSideArgs) do (socketPath: string): - bootClientSide(turn, ds, store, socketPath) + bootClientSide(turn, ds, erisStore, socketPath) during(turn, root, ?DaemonSideArgs) do (socketPath: string): - bootDaemonSide(turn, ds, store, socketPath) + bootDaemonSide(turn, ds, erisStore, socketPath) diff --git a/src/nix_actor.nim.cfg b/src/nix_actor.nim.cfg new file mode 100644 index 000000000000..1dcd32bde059 --- /dev/null +++ b/src/nix_actor.nim.cfg @@ -0,0 +1 @@ +backend:cpp diff --git a/src/nix_actor/daemons.nim b/src/nix_actor/daemons.nim index 8ea436121cb8..b0dfbd6c9d31 100644 --- a/src/nix_actor/daemons.nim +++ b/src/nix_actor/daemons.nim @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -import std/[asyncdispatch, asyncnet, sets, streams, strutils] +import std/[asyncdispatch, asyncnet, sets, strutils] from std/algorithm import sort import eris @@ -9,18 +9,9 @@ import preserves, syndicate from syndicate/protocols/dataspace import Observe import ./protocol, ./sockets -type Value = Preserve[void] - -proc merge(items: varargs[Value]): Value = - # TODO: just a hack, not a proper imlementation - # https://preserves.dev/preserves.html#appendix-merging-values - result = initDictionary() - for e in items: - for (key, val) in e.pairs: - result[key] = val - cannonicalize(result) - -type Observe = dataspace.Observe[Ref] +type + Value = Preserve[void] + Observe = dataspace.Observe[Ref] proc recvError(daemon: Session): Future[string] {.async.} = discard #[typ]# await recvString(daemon) @@ -137,9 +128,7 @@ proc recvLegacyPathAttrs(daemon: Session): Future[AddToStoreAttrs] {.async.} = return info proc addToStore(daemon: Session; store: ErisStore; request: AddToStoreClientAttrs): Future[(string, AddToStoreAttrs)] {.async.} = - let - erisCap = parseCap(request.eris) - stream = newErisStream(store, erisCap) + let erisCap = parseCap(request.eris) await send(daemon, Word wopAddToStore) await send(daemon, request.name) await send(daemon, string request.`ca-method`) diff --git a/src/nix_actor/libnix/libexpr.nim b/src/nix_actor/libnix/libexpr.nim new file mode 100644 index 000000000000..899ec18c7d0f --- /dev/null +++ b/src/nix_actor/libnix/libexpr.nim @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +import ./stdpuspus, ./store + +{.passC: staticExec("pkg-config --cflags nix-expr").} +{.passL: staticExec("pkg-config --libs nix-expr").} + +proc parentDir(path: string): string = + var i = path.high + while path[i] != '/': dec(i) + path[0..i] + +{.passC: "-I" & parentDir(currentSourcePath).} + +type + NixInt* = int64 + NixFloat* = float64 + + ValueKind* {.importcpp: "nix::ValueType", header: "value.hh".} = enum + nThunk, + nInt, + nFloat, + nBool, + nString, + nPath, + nNull, + nAttrs, + nList, + nFunction, + nExternal, + Value* = ValueObj | ValuePtr + ValuePtr* = ptr ValueObj + ValueObj* {.importcpp: "nix::Value", header: "value.hh".} = object + integer*: NixInt + boolean*: bool + string: StringContext + path*: cstring + fpoint*: NixFloat + attrs: Bindings + StringContext = object + s: cstring + Symbol* {.importcpp: "nix::Symbol", header: "symbol-table.hh".} = object + discard + Attr {.importcpp: "nix::Attr", header: "attr-set.hh".} = object + name: Symbol + value: ValuePtr + Bindings = ptr BindginsObj + BindginsObj {.importcpp: "nix::Bindings", header: "attr-set.hh".} = object + discard + +proc kind*(val: Value): ValueKind {.importcpp: "#.type()".} + +proc shallowString*(val: Value): string = + if val.kind != nString: + raise newException(FieldDefect, "Value not an attribute set") + $val.string.s + +proc size(bindings: Bindings): csize_t {.importcpp.} + +proc `[]`(b: Bindings; i: Natural): Attr {.importcpp: "(*#)[#]".} + +iterator pairs*(val: Value): (Symbol, ValuePtr) = + if val.kind != nAttrs: + raise newException(FieldDefect, "Value not an attribute set") + for i in 0..", header: "eval.hh".} = object + discard + +proc newEvalState*(store: Store): EvalState {. + importcpp: "nix::newEvalState(@)", header: "seepuspus.hh".} + +proc parseExprFromString*(state: EvalState; s, basePath: cstring): Expr {. + importcpp: "#->parseExprFromString(@)".} + +proc eval*(state: EvalState; expr: Expr; value: var ValueObj) {. + importcpp: "#->eval(@)".} + +proc forceValueDeep*(state: EvalState; value: var ValueObj) {. + importcpp: "#->forceValueDeep(@)".} + +proc stringView(state: EvalState; sym: Symbol): StringView {. + importcpp: "((std::string_view)#->symbols[#])".} + +proc symbolString*(state: EvalState; sym: Symbol): string = $stringView(state, sym) + +proc initGC*() {.importcpp: "nix::initGC", header: "eval.hh".} diff --git a/src/nix_actor/libnix/main.nim b/src/nix_actor/libnix/main.nim new file mode 100644 index 000000000000..072e97e589e5 --- /dev/null +++ b/src/nix_actor/libnix/main.nim @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +{.passC: staticExec("pkg-config --cflags nix-main").} +{.passL: staticExec("pkg-config --libs nix-main").} + +proc initNix*() {.importcpp: "nix::initNix", header: "shared.hh".} diff --git a/src/nix_actor/libnix/seepuspus.hh b/src/nix_actor/libnix/seepuspus.hh new file mode 100644 index 000000000000..0fef8382352f --- /dev/null +++ b/src/nix_actor/libnix/seepuspus.hh @@ -0,0 +1,21 @@ +#pragma once +#include "eval.hh" + +namespace nix { + + ref newEvalState(ref store) + { + auto searchPath = Strings(); + auto evalState = + #if HAVE_BOEHMGC + std::allocate_shared( + traceable_allocator(), searchPath, store, store) + #else + std::make_shared( + searchPath, store, store) + #endif + ; + return ref(evalState); + } + +} diff --git a/src/nix_actor/libnix/stdpuspus.nim b/src/nix_actor/libnix/stdpuspus.nim new file mode 100644 index 000000000000..171a92585245 --- /dev/null +++ b/src/nix_actor/libnix/stdpuspus.nim @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +type StringView* {.importcpp: "std::string_view", header: "".} = object + +proc toStringView*(s: pointer; count: int): StringView {. + importcpp: "std::string_view(static_cast(#), #)", constructor.} + +proc toStringView*(s: string): StringView {.inline.} = + if s.len == 0: toStringView(nil, 0) + else: toStringView(unsafeAddr s[0], s.len) + +proc toStringView*(buf: openarray[byte]): StringView {.inline.} = + if buf.len == 0: toStringView(nil, 0) + else: toStringView(unsafeAddr buf[0], buf.len) + +proc toStringView*(sv: StringView): StringView {.inline.} = sv + +proc data(sv: StringView): pointer {.importcpp.} +proc size(sv: StringView): csize_t {.importcpp.} + +proc `$`*(sv: StringView): string = + result = newString(sv.size) + copyMem(addr result[0], sv.data, result.len) diff --git a/src/nix_actor/libnix/store.nim b/src/nix_actor/libnix/store.nim new file mode 100644 index 000000000000..a159927cdd6b --- /dev/null +++ b/src/nix_actor/libnix/store.nim @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +{.passC: staticExec("pkg-config --cflags nix-store").} +{.passL: staticExec("pkg-config --libs nix-store").} + +{.passC: "'-DSYSTEM=\"x86_64-linux\"'".} + +type StdString {.importcpp: "std::string", header: "".} = object +proc data(s: StdString): pointer {.importcpp: "#.data()".} +proc len(s: StdString): csize_t {.importcpp: "#.length()".} +proc `$`*(cpp: StdString): string = + result.setLen(cpp.len) + if result.len > 0: + copyMem(addr result[0], cpp.data, result.len) + +type + StorePath {.importcpp: "nix::StorePath", header: "path.hh".} = object + discard + +var nixVersion* {.importc: "nix::nixVersion", header: "globals.hh".}: StdString + +proc isDerivation*(path: StorePath): bool {.importcpp.} + +type + Store* {.importcpp: "nix::ref", header: "store-api.hh".} = object + discard + +proc ensurePath*(store: Store; path: StorePath) {.importcpp.} + +proc openStore*(): Store {.importcpp: "nix::openStore".} diff --git a/src/nix_actor/sockets.nim b/src/nix_actor/sockets.nim index 3d67e7a1859b..a7d85d0ded67 100644 --- a/src/nix_actor/sockets.nim +++ b/src/nix_actor/sockets.nim @@ -33,6 +33,7 @@ const STDERR_RESULT* = 0x52534C54 type WorkerOperation* = enum + wopInvalid = 0, wopIsValidPath = 1, wopHasSubstitutes = 3, wopQueryPathHash = 4, # obsolete diff --git a/src/nix_actor/store.nim b/src/nix_actor/store.nim deleted file mode 100644 index 5808608d1925..000000000000 --- a/src/nix_actor/store.nim +++ /dev/null @@ -1,31 +0,0 @@ -# SPDX-FileCopyrightText: ☭ Emery Hemingway -# SPDX-License-Identifier: Unlicense - -{.passC: staticExec("pkg-config --cflags nix-store").} -{.passL: staticExec("pkg-config --libs nix-store").} - -{.passC: "'-DSYSTEM=\"x86_64-linux\"'".} - -type StdString {.importcpp: "std::string", header: "".} = object -proc data(s: StdString): pointer {.importcpp: "#.data()".} -proc len(s: StdString): csize_t {.importcpp: "#.length()".} -proc `$`*(cpp: StdString): string = - result.setLen(cpp.len) - if result.len > 0: - copyMem(addr result[0], cpp.data, result.len) - -type - StorePath {.importcpp: "nix::StorePath", header: "path.hh".} = object - discard - -var nixVersion* {.importc: "nix::nixVersion", header: "globals.hh".}: StdString - -proc isDerivation*(path: StorePath): bool {.importcpp.} - -type - Store {.importcpp: "nix::ref", header: "store-api.hh".} = object - discard - -proc ensurePath*(store: Store; path: StorePath) {.importcpp.} - -proc openStore*(): Store {.importcpp: "nix::openStore".} -- cgit 1.4.1 From c06f1fd24137fc3716275490d75de134ad8b7db6 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Wed, 26 Jul 2023 10:11:15 +0100 Subject: Ref renamed to Cap --- nix_actor.nimble | 2 +- src/nix_actor.nim | 26 +++++++++++++++++--------- src/nix_actor/clients.nim | 14 +++++++------- src/nix_actor/daemons.nim | 6 +++--- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/nix_actor.nimble b/nix_actor.nimble index 0ab4c42947c1..04245f7f2910 100644 --- a/nix_actor.nimble +++ b/nix_actor.nimble @@ -1,4 +1,4 @@ -version = "20230613" +version = "20230726" author = "Emery Hemingway" description = "Syndicated Nix Actor" license = "Unlicense" diff --git a/src/nix_actor.nim b/src/nix_actor.nim index ce547758664a..81c54f8e8985 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -12,7 +12,7 @@ import ./nix_actor/protocol type Value = Preserve[void] - Observe = dataspace.Observe[Ref] + Observe = dataspace.Observe[Cap] proc toPreserve(val: libexpr.ValueObj | libExpr.ValuePtr; state: EvalState; E = void): Preserve[E] {.gcsafe.} = ## Convert a Nix value to a Preserves value. @@ -75,7 +75,7 @@ proc parseNarinfo(info: var AttrSet; text: string) = else: info[Symbol key] = val.toPreserve -proc narinfo(turn: var Turn; ds: Ref; path: string) = +proc narinfo(turn: var Turn; ds: Cap; path: string) = let client = newAsyncHttpClient() url = "https://cache.nixos.org/" & path & ".narinfo" @@ -109,7 +109,7 @@ proc instantiate(instantiate: Instantiate): Value = var execOutput = strip execProcess(cmd, args = args, options = {poUsePath}) execOutput.toPreserve -proc bootNixFacet(turn: var Turn; ds: Ref): Facet = +proc bootNixFacet(turn: var Turn; ds: Cap): Facet = # let store = openStore() result = inFacet(turn) do (turn: var Turn): @@ -135,8 +135,8 @@ proc bootNixFacet(turn: var Turn; ds: Ref): Facet = ]# type - RefArgs {.preservesDictionary.} = object - dataspace: Ref + CapArgs {.preservesDictionary.} = object + dataspace: Cap ClientSideArgs {.preservesDictionary.} = object `listen-socket`: string DaemonSideArgs {.preservesDictionary.} = object @@ -145,22 +145,30 @@ type main.initNix() libexpr.initGC() -runActor("main") do (root: Ref; turn: var Turn): +runActor("main") do (root: Cap; turn: var Turn): let erisStore = newMemoryStore() nixStore = openStore() nixState = newEvalState(nixStore) connectStdio(root, turn) - during(turn, root, ?RefArgs) do (ds: Ref): + during(turn, root, ?CapArgs) do (ds: Cap): + discard publish(turn, ds, + initRecord("nixVersion", toPreserve($nixVersion.c_str))) + discard bootNixFacet(turn, ds) during(turn, ds, ?Observe(pattern: !Eval) ?? {0: grabLit(), 1: grabDict()}) do (e: string, o: Assertion): var ass = Eval(expr: e) doAssert fromPreserve(ass.options, unpackLiterals(o)) # unused options - ass.result = eval(nixState, ass.expr) - discard publish(turn, ds, ass) + try: + ass.result = eval(nixState, ass.expr) + discard publish(turn, ds, ass) + except CatchableError as err: + stderr.writeLine "failed to evaluate ", ass.expr, ": ", err.msg + except StdException as err: + stderr.writeLine "failed to evaluate ", ass.expr, ": ", err.what during(turn, root, ?ClientSideArgs) do (socketPath: string): bootClientSide(turn, ds, erisStore, socketPath) diff --git a/src/nix_actor/clients.nim b/src/nix_actor/clients.nim index 0ee95eb621a4..b227ac332559 100644 --- a/src/nix_actor/clients.nim +++ b/src/nix_actor/clients.nim @@ -43,7 +43,7 @@ proc completeAddToStore(client: Session; path: string; info: LegacyPathAttrs) {. await send(client, path) await send(client, info) -proc serveClient(facet: Facet; ds: Ref; store: ErisStore; client: Session) {.async.} = +proc serveClient(facet: Facet; ds: Cap; store: ErisStore; client: Session) {.async.} = block: let clientMagic = await recvWord(client) if clientMagic != WORKER_MAGIC_1: @@ -77,10 +77,10 @@ proc serveClient(facet: Facet; ds: Ref; store: ErisStore; client: Session) {.asy let cap = await ingestChunks(client, store) await sendNext(client, $cap & " " & name) let attrsPat = inject(?AddToStoreAttrs, { - "name".toSymbol(Ref): ?name, - "ca-method".toSymbol(Ref): ?caMethod.toSymbol, - "references".toSymbol(Ref): ?storeRefs, - "eris".toSymbol(Ref): ?cap.bytes, + "name".toSymbol(Cap): ?name, + "ca-method".toSymbol(Cap): ?caMethod.toSymbol, + "references".toSymbol(Cap): ?storeRefs, + "eris".toSymbol(Cap): ?cap.bytes, }) # bind AddToStoreAttrs and override with some literal values let pat = PathInfo ? { 0: grab(), 1: attrsPat } @@ -154,7 +154,7 @@ proc serveClient(facet: Facet; ds: Ref; store: ErisStore; client: Session) {.asy await sendWorkEnd(client) close(client.socket) -proc serveClientSide(facet: Facet; ds: Ref; store: ErisStore; listener: AsyncSocket) {.async.} = +proc serveClientSide(facet: Facet; ds: Cap; store: ErisStore; listener: AsyncSocket) {.async.} = while not listener.isClosed: let client = await accept(listener) @@ -163,7 +163,7 @@ proc serveClientSide(facet: Facet; ds: Ref; store: ErisStore; listener: AsyncSoc if not client.isClosed: close(client) -proc bootClientSide*(turn: var Turn; ds: Ref; store: ErisStore; socketPath: string) = +proc bootClientSide*(turn: var Turn; ds: Cap; store: ErisStore; socketPath: string) = let listener = newUnixSocket() onStop(turn.facet) do (turn: var Turn): close(listener) diff --git a/src/nix_actor/daemons.nim b/src/nix_actor/daemons.nim index b0dfbd6c9d31..3bc5591744ed 100644 --- a/src/nix_actor/daemons.nim +++ b/src/nix_actor/daemons.nim @@ -11,7 +11,7 @@ import ./protocol, ./sockets type Value = Preserve[void] - Observe = dataspace.Observe[Ref] + Observe = dataspace.Observe[Cap] proc recvError(daemon: Session): Future[string] {.async.} = discard #[typ]# await recvString(daemon) @@ -153,9 +153,9 @@ proc callDaemon(turn: var Turn; path: string; action: proc (daemon: Session; tur action(daemon, turn) return daemon -proc bootDaemonSide*(turn: var Turn; ds: Ref; store: ErisStore; socketPath: string) = +proc bootDaemonSide*(turn: var Turn; ds: Cap; store: ErisStore; socketPath: string) = - during(turn, ds, ?Observe(pattern: !Missing) ?? {0: grab()}) do (a: Preserve[Ref]): + during(turn, ds, ?Observe(pattern: !Missing) ?? {0: grab()}) do (a: Preserve[Cap]): # cannot use `grabLit` here because an array is a compound # TODO: unpack to a `Pattern` let daemon = callDaemon(turn, socketPath) do (daemon: Session; turn: var Turn): -- cgit 1.4.1 From 8e06628d2113116e86635e97ebceae08edc2ed6c Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sun, 30 Jul 2023 11:37:42 +0100 Subject: Make Nix functions un-Preservable --- nix_actor.nimble | 2 +- src/nix_actor.nim | 27 ++++++++++++++------------- src/nix_actor/libnix/libexpr.nim | 2 ++ src/nix_actor/libnix/stdpuspus.nim | 9 +++++++++ 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/nix_actor.nimble b/nix_actor.nimble index 04245f7f2910..24d40cd7b6d5 100644 --- a/nix_actor.nimble +++ b/nix_actor.nimble @@ -1,4 +1,4 @@ -version = "20230726" +version = "20230730" author = "Emery Hemingway" description = "Syndicated Nix Actor" license = "Unlicense" diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 81c54f8e8985..faa359152da2 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -1,24 +1,25 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -import std/[json, os, osproc, parseutils, strutils, tables] +import std/[json, os, osproc, strutils, tables] import eris/memory_stores import preserves, preserves/jsonhooks import syndicate from syndicate/protocols/dataspace import Observe import ./nix_actor/[clients, daemons] -import ./nix_actor/libnix/[libexpr, main, store] +import ./nix_actor/libnix/[libexpr, main, stdpuspus, store] import ./nix_actor/protocol +var nixVersion {.importcpp: "nix::nixVersion", header: "globals.hh".}: StdString + type Value = Preserve[void] Observe = dataspace.Observe[Cap] -proc toPreserve(val: libexpr.ValueObj | libExpr.ValuePtr; state: EvalState; E = void): Preserve[E] {.gcsafe.} = +proc toPreserve(state: EvalState; val: libexpr.ValueObj | libExpr.ValuePtr; E = void): Preserve[E] {.gcsafe.} = ## Convert a Nix value to a Preserves value. + # See nix::printValueAsJSON case val.kind - of nThunk: - result = initRecord[E]("thunk") of nInt: result = val.integer.toPreserve(E) of nFloat: @@ -33,16 +34,16 @@ proc toPreserve(val: libexpr.ValueObj | libExpr.ValuePtr; state: EvalState; E = result = initRecord[E]("null") of nAttrs: result = initDictionary(E) - for key, val in val.pairs: - result[symbolString(state, key).toSymbol(E)] = val.toPreserve(state, E) + for sym, attr in val.pairs: + let key = symbolString(state, sym).toSymbol(E) + # Nix string to Nim string to Preserves symbol + result[key] = state.toPreserve(attr, E) of nList: result = initSequence(0, E) for e in val.items: - result.sequence.add(e.toPreserve(state, E)) - of nFunction: - result = initRecord[E]("func") - of nExternal: - result = initRecord[E]("external") + result.sequence.add(state.toPreserve(e, E)) + else: + raise newException(ValueError, "cannot preserve " & $val.kind) proc eval(state: EvalState; code: string): Value = ## Evaluate Nix `code` to a Preserves value. @@ -50,7 +51,7 @@ proc eval(state: EvalState; code: string): Value = let expr = state.parseExprFromString(code, getCurrentDir()) state.eval(expr, nixVal) state.forceValueDeep(nixVal) - nixVal.toPreserve(state, void) + state.toPreserve(nixVal, void) proc parseArgs(args: var seq[string]; opts: AttrSet) = for sym, val in opts: diff --git a/src/nix_actor/libnix/libexpr.nim b/src/nix_actor/libnix/libexpr.nim index 899ec18c7d0f..0608dd5ed6c8 100644 --- a/src/nix_actor/libnix/libexpr.nim +++ b/src/nix_actor/libnix/libexpr.nim @@ -51,6 +51,8 @@ type proc kind*(val: Value): ValueKind {.importcpp: "#.type()".} +proc showType*(val: Value): StdString {.importcpp.} + proc shallowString*(val: Value): string = if val.kind != nString: raise newException(FieldDefect, "Value not an attribute set") diff --git a/src/nix_actor/libnix/stdpuspus.nim b/src/nix_actor/libnix/stdpuspus.nim index 171a92585245..719f2f43d3d3 100644 --- a/src/nix_actor/libnix/stdpuspus.nim +++ b/src/nix_actor/libnix/stdpuspus.nim @@ -1,6 +1,15 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense +type StdException* {.importcpp: "std::exception", header: "".} = object + +proc what*(ex: StdException): cstring {.importcpp: "((char *)#.what())", nodecl.} + + +type StdString* {.importcpp: "std::string", header: "".} = object + +proc c_str*(s: StdString): cstring {.importcpp.} + type StringView* {.importcpp: "std::string_view", header: "".} = object proc toStringView*(s: pointer; count: int): StringView {. -- cgit 1.4.1 From 6b6187f508595ce4a8351f8cb56ccc987326e13a Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Thu, 24 Aug 2023 16:16:04 +0100 Subject: Update to Nim-2.0.0 --- nix_actor.nimble | 2 +- shell.nix | 4 +-- src/nix_actor.nim | 72 +++++++++++++++++++++------------------ src/nix_actor/libnix/libexpr.nim | 4 +-- src/nix_actor/libnix/seepuspus.hh | 4 +-- 5 files changed, 46 insertions(+), 40 deletions(-) diff --git a/nix_actor.nimble b/nix_actor.nimble index 24d40cd7b6d5..6443b4287f73 100644 --- a/nix_actor.nimble +++ b/nix_actor.nimble @@ -1,4 +1,4 @@ -version = "20230730" +version = "20230824" author = "Emery Hemingway" description = "Syndicated Nix Actor" license = "Unlicense" diff --git a/shell.nix b/shell.nix index 5d685670df3f..00ec36db0e1e 100644 --- a/shell.nix +++ b/shell.nix @@ -1,4 +1,4 @@ let - flake = builtins.getFlake "syndicate"; + flake = builtins.getFlake "/home/emery/src/syndicate-flake"; pkgs = import { overlays = [ flake.overlays.default ]; }; -in pkgs.nix_actor +in pkgs.nim2Packages.nix_actor diff --git a/src/nix_actor.nim b/src/nix_actor.nim index faa359152da2..aada02bd8bd7 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -143,36 +143,42 @@ type DaemonSideArgs {.preservesDictionary.} = object `daemon-socket`: string -main.initNix() -libexpr.initGC() - -runActor("main") do (root: Cap; turn: var Turn): - let - erisStore = newMemoryStore() - nixStore = openStore() - nixState = newEvalState(nixStore) - connectStdio(root, turn) - - during(turn, root, ?CapArgs) do (ds: Cap): - discard publish(turn, ds, - initRecord("nixVersion", toPreserve($nixVersion.c_str))) - - discard bootNixFacet(turn, ds) - - during(turn, ds, ?Observe(pattern: !Eval) ?? {0: grabLit(), 1: grabDict()}) do (e: string, o: Assertion): - var ass = Eval(expr: e) - doAssert fromPreserve(ass.options, unpackLiterals(o)) - # unused options - try: - ass.result = eval(nixState, ass.expr) - discard publish(turn, ds, ass) - except CatchableError as err: - stderr.writeLine "failed to evaluate ", ass.expr, ": ", err.msg - except StdException as err: - stderr.writeLine "failed to evaluate ", ass.expr, ": ", err.what - - during(turn, root, ?ClientSideArgs) do (socketPath: string): - bootClientSide(turn, ds, erisStore, socketPath) - - during(turn, root, ?DaemonSideArgs) do (socketPath: string): - bootDaemonSide(turn, ds, erisStore, socketPath) +proc runNixActor(nixState: EvalState) = + let erisStore = newMemoryStore() + runActor("nix_actor") do (root: Cap; turn: var Turn): + connectStdio(root, turn) + + let pat = ?CapArgs + during(turn, root, pat) do (ds: Cap): + + discard publish(turn, ds, + initRecord("nixVersion", toPreserve($nixVersion.c_str))) + + discard bootNixFacet(turn, ds) + + let pat = ?Observe(pattern: !Eval) ?? {0: grabLit(), 1: grabDict()} + during(turn, ds, pat) do (e: string, o: Assertion): + var ass = Eval(expr: e) + doAssert fromPreserve(ass.options, unpackLiterals(o)) + # unused options + try: + ass.result = eval(nixState, ass.expr) + discard publish(turn, ds, ass) + except CatchableError as err: + stderr.writeLine "failed to evaluate ", ass.expr, ": ", err.msg + except StdException as err: + stderr.writeLine "failed to evaluate ", ass.expr, ": ", err.what + + during(turn, root, ?ClientSideArgs) do (socketPath: string): + bootClientSide(turn, ds, erisStore, socketPath) + + during(turn, root, ?DaemonSideArgs) do (socketPath: string): + bootDaemonSide(turn, ds, erisStore, socketPath) + +proc main = + initNix() + initGC() + let nixStore = openStore() + runNixActor(newEvalState(nixStore)) + +main() diff --git a/src/nix_actor/libnix/libexpr.nim b/src/nix_actor/libnix/libexpr.nim index 0608dd5ed6c8..d98e20052990 100644 --- a/src/nix_actor/libnix/libexpr.nim +++ b/src/nix_actor/libnix/libexpr.nim @@ -83,11 +83,11 @@ type ExprObj {.importcpp: "nix::Expr", header: "nixexpr.hh".} = object discard Expr* = ptr ExprObj - EvalState* {.importcpp: "nix::ref", header: "eval.hh".} = object + EvalState* {.importcpp: "std::shared_ptr", header: "eval.hh".} = object discard proc newEvalState*(store: Store): EvalState {. - importcpp: "nix::newEvalState(@)", header: "seepuspus.hh".} + importcpp: "nix::newEvalState(@)", header: "seepuspus.hh", constructor.} proc parseExprFromString*(state: EvalState; s, basePath: cstring): Expr {. importcpp: "#->parseExprFromString(@)".} diff --git a/src/nix_actor/libnix/seepuspus.hh b/src/nix_actor/libnix/seepuspus.hh index 0fef8382352f..f1ca69484fdf 100644 --- a/src/nix_actor/libnix/seepuspus.hh +++ b/src/nix_actor/libnix/seepuspus.hh @@ -3,7 +3,7 @@ namespace nix { - ref newEvalState(ref store) + std::shared_ptr newEvalState(ref store) { auto searchPath = Strings(); auto evalState = @@ -15,7 +15,7 @@ namespace nix { searchPath, store, store) #endif ; - return ref(evalState); + return evalState; } } -- cgit 1.4.1 From d22cd1f629e42b28a7e727749d6c8d0c1c2d29c7 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 25 Sep 2023 12:23:14 +0100 Subject: Decrease worker protocol version to 34 --- nix_actor.nimble | 2 +- src/nix_actor/sockets.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nix_actor.nimble b/nix_actor.nimble index 6443b4287f73..21fd165829dd 100644 --- a/nix_actor.nimble +++ b/nix_actor.nimble @@ -1,4 +1,4 @@ -version = "20230824" +version = "20230925" author = "Emery Hemingway" description = "Syndicated Nix Actor" license = "Unlicense" diff --git a/src/nix_actor/sockets.nim b/src/nix_actor/sockets.nim index a7d85d0ded67..13fc093a5308 100644 --- a/src/nix_actor/sockets.nim +++ b/src/nix_actor/sockets.nim @@ -21,7 +21,7 @@ proc `[]=`*[T](attrs: var AttrSet; key: string; val: T) = const WORKER_MAGIC_1* = 0x6E697863 WORKER_MAGIC_2* = 0x6478696F - PROTOCOL_VERSION* = 0x100 or 35 + PROTOCOL_VERSION* = 0x100 or 34 STDERR_NEXT* = 0x6F6C6d67 STDERR_READ* = 0x64617461 -- cgit 1.4.1 From e44368b01fac10912995eea953ff8e5a6b365082 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 9 Oct 2023 19:57:05 +0100 Subject: Add lockfile --- .gitignore | 2 +- Tupfile | 3 +++ Tuprules.tup | 2 ++ lock.json | 1 + nix_actor.nimble | 2 +- shell.nix | 18 +++++++++++++++--- 6 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 Tupfile create mode 100644 lock.json diff --git a/.gitignore b/.gitignore index 7ad627584e63..8454dc769ad1 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -/.direnv +/nim.cfg diff --git a/Tupfile b/Tupfile new file mode 100644 index 000000000000..9368191cbc04 --- /dev/null +++ b/Tupfile @@ -0,0 +1,3 @@ +include_rules +: |> !nim_lk |> | ./ +: lock.json |> !nim_cfg |> | ./ diff --git a/Tuprules.tup b/Tuprules.tup index ab2cf9fc2379..0d08158a620e 100644 --- a/Tuprules.tup +++ b/Tuprules.tup @@ -3,3 +3,5 @@ NIM_FLAGS += --path:$(TUP_CWD)/../eris-nim/src include ../syndicate-nim/depends.tup NIM_FLAGS += --path:$(TUP_CWD)/../syndicate-nim/src + +NIM_GROUPS += $(TUP_CWD)/ diff --git a/lock.json b/lock.json new file mode 100644 index 000000000000..21f963e493c5 --- /dev/null +++ b/lock.json @@ -0,0 +1 @@ +{"depends":[{"method":"fetchzip","packages":["syndicate"],"path":"/nix/store/008s11kkqscfqxs6g29q77c38pnrlppi-source","ref":"20231005","rev":"552e51899c82c0c2f4f466382be7d8e22a1da689","sha256":"1j3k0zlh5z02adhfvb7rdqz8fjzc6gri4v3v1fgcv2h2b7vrf0dg","srcDir":"src","url":"https://git.syndicate-lang.org/ehmry/syndicate-nim/archive/552e51899c82c0c2f4f466382be7d8e22a1da689.tar.gz"},{"method":"fetchzip","packages":["coap"],"path":"/nix/store/pqj933cnw7r7hp46jrpjlwh1yr0jvckp-source","ref":"20230331","rev":"a134213b51a8d250684f2ba26802ffa97fae4ffb","sha256":"1wbix6d8l26nj7m3xinh4m2f27n4ma0yzs3x5lpann2ha0y51k8b","srcDir":"src","url":"https://codeberg.org/eris/nim-coap/archive/a134213b51a8d250684f2ba26802ffa97fae4ffb.tar.gz"},{"method":"fetchzip","packages":["hashlib"],"path":"/nix/store/v03nzlpdgbfxd2zhcnkfbkq01d5kqxcl-source","rev":"84e0247555e4488594975900401baaf5bbbfb53","sha256":"1pfczsv8kl36qpv543f93d2y2vgz2acckssfap7l51s2x62m6qwx","srcDir":"","url":"https://github.com/khchen/hashlib/archive/84e0247555e4488594975900401baaf5bbbfb53.tar.gz"},{"method":"fetchzip","packages":["preserves"],"path":"/nix/store/vx6ihnickx7d5lwy69i8k7fsjicv33r3-source","ref":"20230914","rev":"c915accf7d2a36ca1f323e2f02e2df7375e815f1","sha256":"11rlcbs9mvk335ibkbj8fk9aslhmnlaiqhcsjpp5n04k447sr7nx","srcDir":"src","url":"https://git.syndicate-lang.org/ehmry/preserves-nim/archive/c915accf7d2a36ca1f323e2f02e2df7375e815f1.tar.gz"},{"method":"fetchzip","packages":["eris"],"path":"/nix/store/lxa6ba8r9hhs06k6f2iyznwjxix1klv1-source","ref":"20230823","rev":"49d8117367d3530533dc1d6a9111ddd134b08b1e","sha256":"0lq9a04cayf04nnhn0gvp5phlij0cis38v7cz7jmgks2xvz1bcbr","srcDir":"src","url":"https://codeberg.org/eris/nim-eris/archive/49d8117367d3530533dc1d6a9111ddd134b08b1e.tar.gz"},{"method":"fetchzip","packages":["cbor"],"path":"/nix/store/70cqa9s36dqnmsf179cn9psj77jhqi1l-source","ref":"20230619","rev":"a4a1affd45ba90bea24e08733ae2bd02fe058166","sha256":"005ib6im97x9pdbg6p0fy58zpdwdbkpmilxa8nhrrb1hnpjzz90p","srcDir":"src","url":"https://git.sr.ht/~ehmry/nim_cbor/archive/a4a1affd45ba90bea24e08733ae2bd02fe058166.tar.gz"},{"method":"fetchzip","packages":["freedesktop_org"],"path":"/nix/store/98wncmx58cfnhv3y96lzwm22zvyk9b1h-source","ref":"20230210","rev":"fb04d0862aca4be2edcc0eafa94b1840030231c8","sha256":"0wj5m09x1pr36gv8p5r72p6l3wwl01y8scpnlzx7q0h5ij6jaj6s","srcDir":"src","url":"https://git.sr.ht/~ehmry/freedesktop_org/archive/fb04d0862aca4be2edcc0eafa94b1840030231c8.tar.gz"},{"method":"fetchzip","packages":["configparser"],"path":"/nix/store/4zl5v7i6cj3f9sayvsjcx2h20lqwr9a6-source","ref":"newSection","rev":"695f1285d63f1954c25eb1f42798d90fa7bcbe14","sha256":"0b0pb5i0kir130ia2zf8zcgdz8awms161i6p83ri3nbgibbjnr37","srcDir":"src","url":"https://github.com/ehmry/nim-configparser/archive/695f1285d63f1954c25eb1f42798d90fa7bcbe14.tar.gz"},{"method":"fetchzip","packages":["tkrzw"],"path":"/nix/store/4x9wxyli4dy719svg1zaww0c0b3xckp0-source","ref":"20220922","rev":"efd87edb7b063182c1a1fa018006a87b515d589b","sha256":"1h0sdvai4gkkz48xfh67wa1xz2k8bkkba8q6snnbllmhmywd9apb","srcDir":"src","url":"https://git.sr.ht/~ehmry/nim-tkrzw/archive/efd87edb7b063182c1a1fa018006a87b515d589b.tar.gz"},{"method":"fetchzip","packages":["getdns"],"path":"/nix/store/x9xmn7w4k6jg8nv5bnx148ibhnsfh362-source","ref":"20221222","rev":"c73cbe288d9f9480586b8fa87f6d794ffb6a6ce6","sha256":"1sbgx2x51szr22i72n7c8jglnfmr8m7y7ga0v85d58fwadiv7g6b","srcDir":"src","url":"https://git.sr.ht/~ehmry/getdns-nim/archive/c73cbe288d9f9480586b8fa87f6d794ffb6a6ce6.tar.gz"},{"method":"fetchzip","packages":["nimcrypto"],"path":"/nix/store/zyr8zwh7vaiycn1s4r8cxwc71f2k5l0h-source","ref":"traditional-api","rev":"602c5d20c69c76137201b5d41f788f72afb95aa8","sha256":"1dmdmgb6b9m5f8dyxk781nnd61dsk3hdxqks7idk9ncnpj9fng65","srcDir":"","url":"https://github.com/cheatfate/nimcrypto/archive/602c5d20c69c76137201b5d41f788f72afb95aa8.tar.gz"},{"method":"fetchzip","packages":["taps"],"path":"/nix/store/did1li0xk9qih80pvxqhjc4np3ijlfjj-source","ref":"20230331","rev":"4f9c9972d74eb39c662b43ed79d761e109bf00f1","sha256":"12qsizmisr1q0q4x37c5q6gmnqb5mp0bid7s3jlcsjvhc4jw2q57","srcDir":"src","url":"https://git.sr.ht/~ehmry/nim_taps/archive/4f9c9972d74eb39c662b43ed79d761e109bf00f1.tar.gz"},{"method":"fetchzip","packages":["base32"],"path":"/nix/store/qcnchjsak3hyn4c6r0zd6qvm7j8y1747-source","ref":"0.1.3","rev":"f541038fbe49fdb118cc2002d29824b9fc4bfd61","sha256":"16gh1ifp9hslsg0is0v1ya7rxqfhq5hjqzc3pfdqvcgibp5ybh06","srcDir":"","url":"https://github.com/OpenSystemsLab/base32.nim/archive/f541038fbe49fdb118cc2002d29824b9fc4bfd61.tar.gz"},{"method":"fetchzip","packages":["npeg"],"path":"/nix/store/ffkxmjmigfs7zhhiiqm0iw2c34smyciy-source","ref":"1.2.1","rev":"26d62fdc40feb84c6533956dc11d5ee9ea9b6c09","sha256":"0xpzifjkfp49w76qmaylan8q181bs45anmp46l4bwr3lkrr7bpwh","srcDir":"src","url":"https://github.com/zevv/npeg/archive/26d62fdc40feb84c6533956dc11d5ee9ea9b6c09.tar.gz"}]} diff --git a/nix_actor.nimble b/nix_actor.nimble index 21fd165829dd..f532d8258953 100644 --- a/nix_actor.nimble +++ b/nix_actor.nimble @@ -5,4 +5,4 @@ license = "Unlicense" srcDir = "src" bin = @["nix_actor"] -requires "nim >= 1.6.10", "syndicate >= 20230530" +requires "nim >= 1.6.10", "syndicate >= 20231005", "eris >= 20230823" diff --git a/shell.nix b/shell.nix index 00ec36db0e1e..656b0393b32b 100644 --- a/shell.nix +++ b/shell.nix @@ -1,4 +1,16 @@ +{ pkgs ? import { } }: + let - flake = builtins.getFlake "/home/emery/src/syndicate-flake"; - pkgs = import { overlays = [ flake.overlays.default ]; }; -in pkgs.nim2Packages.nix_actor + nix' = pkgs.nix.overrideAttrs (final: prev: { + src = pkgs.fetchFromGitHub { + owner = "NixOS"; + repo = "nix"; + rev = "2.13.3"; + hash = "sha256-jUc2ccTR8f6MGY2pUKgujm+lxSPNGm/ZAP+toX+nMNc="; + }; + }); +in pkgs.nim2Packages.buildNimPackage { + name = "dummy"; + nativeBuildInputs = [ pkgs.pkg-config ]; + buildInputs = [ pkgs.boost nix' ]; +} -- cgit 1.4.1 From 5483b54158b0d5565bc80e6b2ab6e874da071cf6 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 9 Oct 2023 19:57:40 +0100 Subject: Update syndicate imports --- nix_actor.nimble | 2 +- src/nix_actor.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nix_actor.nimble b/nix_actor.nimble index f532d8258953..48c8f6e7fcc6 100644 --- a/nix_actor.nimble +++ b/nix_actor.nimble @@ -1,4 +1,4 @@ -version = "20230925" +version = "20231009" author = "Emery Hemingway" description = "Syndicated Nix Actor" license = "Unlicense" diff --git a/src/nix_actor.nim b/src/nix_actor.nim index aada02bd8bd7..a5a6bfc33bf0 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -4,7 +4,7 @@ import std/[json, os, osproc, strutils, tables] import eris/memory_stores import preserves, preserves/jsonhooks -import syndicate +import syndicate, syndicate/relays from syndicate/protocols/dataspace import Observe import ./nix_actor/[clients, daemons] import ./nix_actor/libnix/[libexpr, main, stdpuspus, store] -- cgit 1.4.1 From 576157b901afba7f78e9b7d98c174fcbca2a788b Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Fri, 8 Dec 2023 08:24:29 +0200 Subject: Use the (un)stable Nix C bindings --- Tupfile | 1 - lock.json | 172 ++++++++++++++++++- nix_actor.nimble | 2 +- protocol.prs | 2 +- shell.nix | 9 +- src/nix_actor.nim | 229 +++++++------------------ src/nix_actor.nim.cfg | 1 - src/nix_actor/daemons.nim | 24 +-- src/nix_actor/nix_api.nim | 58 +++++++ src/nix_actor/nix_api_expr.nim | 28 +++ src/nix_actor/nix_api_store.nim | 26 +++ src/nix_actor/nix_api_util.nim | 35 ++++ src/nix_actor/nix_api_value.nim | 72 ++++++++ src/nix_actor/old.nim | 369 ++++++++++++++++++++++++++++++++++++++++ src/nix_actor/protocol.nim | 2 +- 15 files changed, 839 insertions(+), 191 deletions(-) delete mode 100644 src/nix_actor.nim.cfg create mode 100644 src/nix_actor/nix_api.nim create mode 100644 src/nix_actor/nix_api_expr.nim create mode 100644 src/nix_actor/nix_api_store.nim create mode 100644 src/nix_actor/nix_api_util.nim create mode 100644 src/nix_actor/nix_api_value.nim create mode 100644 src/nix_actor/old.nim diff --git a/Tupfile b/Tupfile index 9368191cbc04..28c450e9095e 100644 --- a/Tupfile +++ b/Tupfile @@ -1,3 +1,2 @@ include_rules -: |> !nim_lk |> | ./ : lock.json |> !nim_cfg |> | ./ diff --git a/lock.json b/lock.json index 21f963e493c5..08f588a19423 100644 --- a/lock.json +++ b/lock.json @@ -1 +1,171 @@ -{"depends":[{"method":"fetchzip","packages":["syndicate"],"path":"/nix/store/008s11kkqscfqxs6g29q77c38pnrlppi-source","ref":"20231005","rev":"552e51899c82c0c2f4f466382be7d8e22a1da689","sha256":"1j3k0zlh5z02adhfvb7rdqz8fjzc6gri4v3v1fgcv2h2b7vrf0dg","srcDir":"src","url":"https://git.syndicate-lang.org/ehmry/syndicate-nim/archive/552e51899c82c0c2f4f466382be7d8e22a1da689.tar.gz"},{"method":"fetchzip","packages":["coap"],"path":"/nix/store/pqj933cnw7r7hp46jrpjlwh1yr0jvckp-source","ref":"20230331","rev":"a134213b51a8d250684f2ba26802ffa97fae4ffb","sha256":"1wbix6d8l26nj7m3xinh4m2f27n4ma0yzs3x5lpann2ha0y51k8b","srcDir":"src","url":"https://codeberg.org/eris/nim-coap/archive/a134213b51a8d250684f2ba26802ffa97fae4ffb.tar.gz"},{"method":"fetchzip","packages":["hashlib"],"path":"/nix/store/v03nzlpdgbfxd2zhcnkfbkq01d5kqxcl-source","rev":"84e0247555e4488594975900401baaf5bbbfb53","sha256":"1pfczsv8kl36qpv543f93d2y2vgz2acckssfap7l51s2x62m6qwx","srcDir":"","url":"https://github.com/khchen/hashlib/archive/84e0247555e4488594975900401baaf5bbbfb53.tar.gz"},{"method":"fetchzip","packages":["preserves"],"path":"/nix/store/vx6ihnickx7d5lwy69i8k7fsjicv33r3-source","ref":"20230914","rev":"c915accf7d2a36ca1f323e2f02e2df7375e815f1","sha256":"11rlcbs9mvk335ibkbj8fk9aslhmnlaiqhcsjpp5n04k447sr7nx","srcDir":"src","url":"https://git.syndicate-lang.org/ehmry/preserves-nim/archive/c915accf7d2a36ca1f323e2f02e2df7375e815f1.tar.gz"},{"method":"fetchzip","packages":["eris"],"path":"/nix/store/lxa6ba8r9hhs06k6f2iyznwjxix1klv1-source","ref":"20230823","rev":"49d8117367d3530533dc1d6a9111ddd134b08b1e","sha256":"0lq9a04cayf04nnhn0gvp5phlij0cis38v7cz7jmgks2xvz1bcbr","srcDir":"src","url":"https://codeberg.org/eris/nim-eris/archive/49d8117367d3530533dc1d6a9111ddd134b08b1e.tar.gz"},{"method":"fetchzip","packages":["cbor"],"path":"/nix/store/70cqa9s36dqnmsf179cn9psj77jhqi1l-source","ref":"20230619","rev":"a4a1affd45ba90bea24e08733ae2bd02fe058166","sha256":"005ib6im97x9pdbg6p0fy58zpdwdbkpmilxa8nhrrb1hnpjzz90p","srcDir":"src","url":"https://git.sr.ht/~ehmry/nim_cbor/archive/a4a1affd45ba90bea24e08733ae2bd02fe058166.tar.gz"},{"method":"fetchzip","packages":["freedesktop_org"],"path":"/nix/store/98wncmx58cfnhv3y96lzwm22zvyk9b1h-source","ref":"20230210","rev":"fb04d0862aca4be2edcc0eafa94b1840030231c8","sha256":"0wj5m09x1pr36gv8p5r72p6l3wwl01y8scpnlzx7q0h5ij6jaj6s","srcDir":"src","url":"https://git.sr.ht/~ehmry/freedesktop_org/archive/fb04d0862aca4be2edcc0eafa94b1840030231c8.tar.gz"},{"method":"fetchzip","packages":["configparser"],"path":"/nix/store/4zl5v7i6cj3f9sayvsjcx2h20lqwr9a6-source","ref":"newSection","rev":"695f1285d63f1954c25eb1f42798d90fa7bcbe14","sha256":"0b0pb5i0kir130ia2zf8zcgdz8awms161i6p83ri3nbgibbjnr37","srcDir":"src","url":"https://github.com/ehmry/nim-configparser/archive/695f1285d63f1954c25eb1f42798d90fa7bcbe14.tar.gz"},{"method":"fetchzip","packages":["tkrzw"],"path":"/nix/store/4x9wxyli4dy719svg1zaww0c0b3xckp0-source","ref":"20220922","rev":"efd87edb7b063182c1a1fa018006a87b515d589b","sha256":"1h0sdvai4gkkz48xfh67wa1xz2k8bkkba8q6snnbllmhmywd9apb","srcDir":"src","url":"https://git.sr.ht/~ehmry/nim-tkrzw/archive/efd87edb7b063182c1a1fa018006a87b515d589b.tar.gz"},{"method":"fetchzip","packages":["getdns"],"path":"/nix/store/x9xmn7w4k6jg8nv5bnx148ibhnsfh362-source","ref":"20221222","rev":"c73cbe288d9f9480586b8fa87f6d794ffb6a6ce6","sha256":"1sbgx2x51szr22i72n7c8jglnfmr8m7y7ga0v85d58fwadiv7g6b","srcDir":"src","url":"https://git.sr.ht/~ehmry/getdns-nim/archive/c73cbe288d9f9480586b8fa87f6d794ffb6a6ce6.tar.gz"},{"method":"fetchzip","packages":["nimcrypto"],"path":"/nix/store/zyr8zwh7vaiycn1s4r8cxwc71f2k5l0h-source","ref":"traditional-api","rev":"602c5d20c69c76137201b5d41f788f72afb95aa8","sha256":"1dmdmgb6b9m5f8dyxk781nnd61dsk3hdxqks7idk9ncnpj9fng65","srcDir":"","url":"https://github.com/cheatfate/nimcrypto/archive/602c5d20c69c76137201b5d41f788f72afb95aa8.tar.gz"},{"method":"fetchzip","packages":["taps"],"path":"/nix/store/did1li0xk9qih80pvxqhjc4np3ijlfjj-source","ref":"20230331","rev":"4f9c9972d74eb39c662b43ed79d761e109bf00f1","sha256":"12qsizmisr1q0q4x37c5q6gmnqb5mp0bid7s3jlcsjvhc4jw2q57","srcDir":"src","url":"https://git.sr.ht/~ehmry/nim_taps/archive/4f9c9972d74eb39c662b43ed79d761e109bf00f1.tar.gz"},{"method":"fetchzip","packages":["base32"],"path":"/nix/store/qcnchjsak3hyn4c6r0zd6qvm7j8y1747-source","ref":"0.1.3","rev":"f541038fbe49fdb118cc2002d29824b9fc4bfd61","sha256":"16gh1ifp9hslsg0is0v1ya7rxqfhq5hjqzc3pfdqvcgibp5ybh06","srcDir":"","url":"https://github.com/OpenSystemsLab/base32.nim/archive/f541038fbe49fdb118cc2002d29824b9fc4bfd61.tar.gz"},{"method":"fetchzip","packages":["npeg"],"path":"/nix/store/ffkxmjmigfs7zhhiiqm0iw2c34smyciy-source","ref":"1.2.1","rev":"26d62fdc40feb84c6533956dc11d5ee9ea9b6c09","sha256":"0xpzifjkfp49w76qmaylan8q181bs45anmp46l4bwr3lkrr7bpwh","srcDir":"src","url":"https://github.com/zevv/npeg/archive/26d62fdc40feb84c6533956dc11d5ee9ea9b6c09.tar.gz"}]} +{ + "depends": [ + { + "method": "fetchzip", + "packages": [ + "base32" + ], + "path": "/nix/store/qcnchjsak3hyn4c6r0zd6qvm7j8y1747-source", + "ref": "0.1.3", + "rev": "f541038fbe49fdb118cc2002d29824b9fc4bfd61", + "sha256": "16gh1ifp9hslsg0is0v1ya7rxqfhq5hjqzc3pfdqvcgibp5ybh06", + "srcDir": "", + "url": "https://github.com/OpenSystemsLab/base32.nim/archive/f541038fbe49fdb118cc2002d29824b9fc4bfd61.tar.gz" + }, + { + "method": "fetchzip", + "packages": [ + "cbor" + ], + "path": "/nix/store/70cqa9s36dqnmsf179cn9psj77jhqi1l-source", + "ref": "20230619", + "rev": "a4a1affd45ba90bea24e08733ae2bd02fe058166", + "sha256": "005ib6im97x9pdbg6p0fy58zpdwdbkpmilxa8nhrrb1hnpjzz90p", + "srcDir": "src", + "url": "https://git.sr.ht/~ehmry/nim_cbor/archive/a4a1affd45ba90bea24e08733ae2bd02fe058166.tar.gz" + }, + { + "method": "fetchzip", + "packages": [ + "coap" + ], + "path": "/nix/store/pqj933cnw7r7hp46jrpjlwh1yr0jvckp-source", + "ref": "20230331", + "rev": "a134213b51a8d250684f2ba26802ffa97fae4ffb", + "sha256": "1wbix6d8l26nj7m3xinh4m2f27n4ma0yzs3x5lpann2ha0y51k8b", + "srcDir": "src", + "url": "https://codeberg.org/eris/nim-coap/archive/a134213b51a8d250684f2ba26802ffa97fae4ffb.tar.gz" + }, + { + "method": "fetchzip", + "packages": [ + "configparser" + ], + "path": "/nix/store/4zl5v7i6cj3f9sayvsjcx2h20lqwr9a6-source", + "ref": "newSection", + "rev": "695f1285d63f1954c25eb1f42798d90fa7bcbe14", + "sha256": "0b0pb5i0kir130ia2zf8zcgdz8awms161i6p83ri3nbgibbjnr37", + "srcDir": "src", + "url": "https://github.com/ehmry/nim-configparser/archive/695f1285d63f1954c25eb1f42798d90fa7bcbe14.tar.gz" + }, + { + "method": "fetchzip", + "packages": [ + "eris" + ], + "path": "/nix/store/lxa6ba8r9hhs06k6f2iyznwjxix1klv1-source", + "ref": "20230823", + "rev": "49d8117367d3530533dc1d6a9111ddd134b08b1e", + "sha256": "0lq9a04cayf04nnhn0gvp5phlij0cis38v7cz7jmgks2xvz1bcbr", + "srcDir": "src", + "url": "https://codeberg.org/eris/nim-eris/archive/49d8117367d3530533dc1d6a9111ddd134b08b1e.tar.gz" + }, + { + "method": "fetchzip", + "packages": [ + "freedesktop_org" + ], + "path": "/nix/store/98wncmx58cfnhv3y96lzwm22zvyk9b1h-source", + "ref": "20230210", + "rev": "fb04d0862aca4be2edcc0eafa94b1840030231c8", + "sha256": "0wj5m09x1pr36gv8p5r72p6l3wwl01y8scpnlzx7q0h5ij6jaj6s", + "srcDir": "src", + "url": "https://git.sr.ht/~ehmry/freedesktop_org/archive/fb04d0862aca4be2edcc0eafa94b1840030231c8.tar.gz" + }, + { + "method": "fetchzip", + "packages": [ + "getdns" + ], + "path": "/nix/store/x9xmn7w4k6jg8nv5bnx148ibhnsfh362-source", + "ref": "20221222", + "rev": "c73cbe288d9f9480586b8fa87f6d794ffb6a6ce6", + "sha256": "1sbgx2x51szr22i72n7c8jglnfmr8m7y7ga0v85d58fwadiv7g6b", + "srcDir": "src", + "url": "https://git.sr.ht/~ehmry/getdns-nim/archive/c73cbe288d9f9480586b8fa87f6d794ffb6a6ce6.tar.gz" + }, + { + "method": "fetchzip", + "packages": [ + "hashlib" + ], + "path": "/nix/store/fav82xdbicvlk34nmcbl89zx99lr3mbs-source", + "rev": "f9455d4be988e14e3dc7933eb7cc7d7c4820b7ac", + "sha256": "1sx6j952lj98629qfgr7ds5aipyw9d6lldcnnqs205wpj4pkcjb3", + "srcDir": "", + "url": "https://github.com/ehmry/hashlib/archive/f9455d4be988e14e3dc7933eb7cc7d7c4820b7ac.tar.gz" + }, + { + "method": "fetchzip", + "packages": [ + "nimcrypto" + ], + "path": "/nix/store/zyr8zwh7vaiycn1s4r8cxwc71f2k5l0h-source", + "ref": "traditional-api", + "rev": "602c5d20c69c76137201b5d41f788f72afb95aa8", + "sha256": "1dmdmgb6b9m5f8dyxk781nnd61dsk3hdxqks7idk9ncnpj9fng65", + "srcDir": "", + "url": "https://github.com/cheatfate/nimcrypto/archive/602c5d20c69c76137201b5d41f788f72afb95aa8.tar.gz" + }, + { + "method": "fetchzip", + "packages": [ + "npeg" + ], + "path": "/nix/store/ffkxmjmigfs7zhhiiqm0iw2c34smyciy-source", + "ref": "1.2.1", + "rev": "26d62fdc40feb84c6533956dc11d5ee9ea9b6c09", + "sha256": "0xpzifjkfp49w76qmaylan8q181bs45anmp46l4bwr3lkrr7bpwh", + "srcDir": "src", + "url": "https://github.com/zevv/npeg/archive/26d62fdc40feb84c6533956dc11d5ee9ea9b6c09.tar.gz" + }, + { + "method": "fetchzip", + "packages": [ + "preserves" + ], + "path": "/nix/store/fmb2yckksz7iv3qdkk5gk1j060kppkq9-source", + "ref": "20231102", + "rev": "4faeb766dc3945bcfacaa1a836ef6ab29b20ceb0", + "sha256": "1a3g5bk1l1h250q3p6sqv6r1lpsplp330qqyp48r0i4a5r0jksq3", + "srcDir": "src", + "url": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/4faeb766dc3945bcfacaa1a836ef6ab29b20ceb0.tar.gz" + }, + { + "method": "fetchzip", + "packages": [ + "syndicate" + ], + "path": "/nix/store/nhpvl223vbzdrlzikw7pgyfxs344w7ma-source", + "ref": "20231108", + "rev": "095418032180e360ea27ec7fcd63193944b68e2c", + "sha256": "09pbml2chzz0v5zpz67fs7raj0mfmg8qrih2vz85xxc51h7ncqvw", + "srcDir": "src", + "url": "https://git.syndicate-lang.org/ehmry/syndicate-nim/archive/095418032180e360ea27ec7fcd63193944b68e2c.tar.gz" + }, + { + "method": "fetchzip", + "packages": [ + "taps" + ], + "path": "/nix/store/did1li0xk9qih80pvxqhjc4np3ijlfjj-source", + "ref": "20230331", + "rev": "4f9c9972d74eb39c662b43ed79d761e109bf00f1", + "sha256": "12qsizmisr1q0q4x37c5q6gmnqb5mp0bid7s3jlcsjvhc4jw2q57", + "srcDir": "src", + "url": "https://git.sr.ht/~ehmry/nim_taps/archive/4f9c9972d74eb39c662b43ed79d761e109bf00f1.tar.gz" + }, + { + "method": "fetchzip", + "packages": [ + "tkrzw" + ], + "path": "/nix/store/4x9wxyli4dy719svg1zaww0c0b3xckp0-source", + "ref": "20220922", + "rev": "efd87edb7b063182c1a1fa018006a87b515d589b", + "sha256": "1h0sdvai4gkkz48xfh67wa1xz2k8bkkba8q6snnbllmhmywd9apb", + "srcDir": "src", + "url": "https://git.sr.ht/~ehmry/nim-tkrzw/archive/efd87edb7b063182c1a1fa018006a87b515d589b.tar.gz" + } + ] +} diff --git a/nix_actor.nimble b/nix_actor.nimble index 48c8f6e7fcc6..ad684e5e370a 100644 --- a/nix_actor.nimble +++ b/nix_actor.nimble @@ -1,4 +1,4 @@ -version = "20231009" +version = "20231208" author = "Emery Hemingway" description = "Syndicated Nix Actor" license = "Unlicense" diff --git a/protocol.prs b/protocol.prs index 335f54915b79..2502aed92a0b 100644 --- a/protocol.prs +++ b/protocol.prs @@ -10,7 +10,7 @@ Realise = . Instantiate = . -Eval = . +Eval = . Narinfo = . diff --git a/shell.nix b/shell.nix index 656b0393b32b..75d225f97474 100644 --- a/shell.nix +++ b/shell.nix @@ -3,14 +3,15 @@ let nix' = pkgs.nix.overrideAttrs (final: prev: { src = pkgs.fetchFromGitHub { - owner = "NixOS"; + owner = "tweag"; repo = "nix"; - rev = "2.13.3"; - hash = "sha256-jUc2ccTR8f6MGY2pUKgujm+lxSPNGm/ZAP+toX+nMNc="; + rev = "nix-c-bindings"; + hash = "sha256-xOyU79lsz0THOj1LccfsDS45089n2DhlkWxaJFeKriY="; }; }); -in pkgs.nim2Packages.buildNimPackage { +in pkgs.buildNimPackage { name = "dummy"; nativeBuildInputs = [ pkgs.pkg-config ]; buildInputs = [ pkgs.boost nix' ]; + lockFile = ./lock.json; } diff --git a/src/nix_actor.nim b/src/nix_actor.nim index a5a6bfc33bf0..b4e58e909674 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -1,184 +1,81 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -import std/[json, os, osproc, strutils, tables] -import eris/memory_stores -import preserves, preserves/jsonhooks -import syndicate, syndicate/relays -from syndicate/protocols/dataspace import Observe -import ./nix_actor/[clients, daemons] -import ./nix_actor/libnix/[libexpr, main, stdpuspus, store] +import std/[os, strutils, tables] +import preserves, syndicate, syndicate/relays +# from syndicate/protocols/dataspace import Observe +import ./nix_actor/[nix_api, nix_api_value] import ./nix_actor/protocol -var nixVersion {.importcpp: "nix::nixVersion", header: "globals.hh".}: StdString - -type - Value = Preserve[void] - Observe = dataspace.Observe[Cap] - -proc toPreserve(state: EvalState; val: libexpr.ValueObj | libExpr.ValuePtr; E = void): Preserve[E] {.gcsafe.} = - ## Convert a Nix value to a Preserves value. - # See nix::printValueAsJSON - case val.kind - of nInt: - result = val.integer.toPreserve(E) - of nFloat: - result = val.fpoint.toPreserve(E) - of nBool: - result = val.boolean.toPreserve(E) - of nString: - result = val.shallowString.toPreserve(E) - of nPath: - result = toSymbol($val.path, E) - of nNull: +proc toPreserve(state: State; value: Value; E = void): Preserve[E] {.gcsafe.} = + var ctx: NixContext + stderr.writeLine get_type(ctx, value).int + case get_type(ctx, value) + of NIX_TYPE_THUNK: raiseAssert "cannot preserve thunk" + of NIX_TYPE_INT: + result = getInt(ctx, value).toPreserve(E) + of NIX_TYPE_FLOAT: + result = getFloat(ctx, value).toPreserve(E) + of NIX_TYPE_BOOL: + result = getBool(ctx, value).toPreserve(E) + of NIX_TYPE_STRING: + result = ($getString(ctx, value)).toPreserve(E) + of NIX_TYPE_PATH: + result = ($getPathString(ctx, value)).toPreserve(E) + of NIX_TYPE_NULL: result = initRecord[E]("null") - of nAttrs: + of NIX_TYPE_ATTRS: result = initDictionary(E) - for sym, attr in val.pairs: - let key = symbolString(state, sym).toSymbol(E) - # Nix string to Nim string to Preserves symbol - result[key] = state.toPreserve(attr, E) - of nList: - result = initSequence(0, E) - for e in val.items: - result.sequence.add(state.toPreserve(e, E)) - else: - raise newException(ValueError, "cannot preserve " & $val.kind) - -proc eval(state: EvalState; code: string): Value = - ## Evaluate Nix `code` to a Preserves value. - var nixVal: libexpr.ValueObj - let expr = state.parseExprFromString(code, getCurrentDir()) - state.eval(expr, nixVal) - state.forceValueDeep(nixVal) - state.toPreserve(nixVal, void) - -proc parseArgs(args: var seq[string]; opts: AttrSet) = - for sym, val in opts: - add(args, "--" & $sym) - if not val.isString "": - var js: JsonNode - if fromPreserve(js, val): add(args, $js) - else: stderr.writeLine "invalid option --", sym, " ", val - -#[ -proc parseNarinfo(info: var AttrSet; text: string) = - var - key, val: string - off: int - while off < len(text): - off = off + parseUntil(text, key, ':', off) + 1 - off = off + skipWhitespace(text, off) - off = off + parseUntil(text, val, '\n', off) + 1 - if key != "" and val != "": - if allCharsInSet(val, Digits): - info[Symbol key] = val.parsePreserves - else: - info[Symbol key] = val.toPreserve - -proc narinfo(turn: var Turn; ds: Cap; path: string) = - let - client = newAsyncHttpClient() - url = "https://cache.nixos.org/" & path & ".narinfo" - futGet = get(client, url) - addCallback(futGet, turn) do (turn: var Turn): - let resp = read(futGet) - if code(resp) != Http200: - close(client) - else: - let futBody = body(resp) - addCallback(futBody, turn) do (turn: var Turn): - close(client) - var narinfo = Narinfo(path: path) - parseNarinfo(narinfo.info, read(futBody)) - discard publish(turn, ds, narinfo) -]# # I never link to openssl if I can avoid it. - -proc build(spec: string): Build = - var execOutput = execProcess("nix", args = ["build", "--json", "--no-link", spec], options = {poUsePath}) - var js = parseJson(execOutput) - Build(input: spec, output: js[0].toPreserve) - -proc realise(realise: Realise): seq[string] = - var execlines = execProcess("nix-store", args = ["--realize", realise.drv], options = {poUsePath}) - split(strip(execlines), '\n') - -proc instantiate(instantiate: Instantiate): Value = - const cmd = "nix-instantiate" - var args = @["--expr", instantiate.expr] - parseArgs(args, instantiate.options) - var execOutput = strip execProcess(cmd, args = args, options = {poUsePath}) - execOutput.toPreserve - -proc bootNixFacet(turn: var Turn; ds: Cap): Facet = - # let store = openStore() - result = inFacet(turn) do (turn: var Turn): - - during(turn, ds, ?Observe(pattern: !Build) ?? {0: grabLit()}) do (spec: string): - discard publish(turn, ds, build(spec)) - - during(turn, ds, ?Observe(pattern: !Realise) ?? {0: grabLit()}) do (drvPath: string): - var ass = Realise(drv: drvPath) - ass.outputs = realise(ass) - discard publish(turn, ds, ass) - - during(turn, ds, ?Observe(pattern: !Instantiate) ?? {0: grabLit(), 1: grabDict()}) do (e: string, o: Value): - var ass = Instantiate(expr: e) - if not fromPreserve(ass.options, unpackLiterals(o)): - stderr.writeLine "invalid options ", o - else: - ass.result = instantiate(ass) - discard publish(turn, ds, ass) - - #[ - during(turn, ds, ?Observe(pattern: !Narinfo) ?? {0: grabLit()}) do (path: string): - narinfo(turn, ds, path) - ]# + let n = getAttrsSize(ctx, value) + var i: cuint + while i < n: + var (key, val) = get_attr_byidx(ctx, value, state, i) + inc(i) + result[toSymbol($key, E)] = toPreserve(state, val, E) + stderr.writeLine(result) + # close(val) + of NIX_TYPE_LIST: + let n = getListSize(ctx, value) + result = initSequence(n, E) + var i: cuint + while i < n: + var val = getListByIdx(ctx, value, state, i) + result[i] = toPreserve(state, val, E) + inc(i) + # close(val) + of NIX_TYPE_FUNCTION, NIX_TYPE_EXTERNAL: + raiseAssert "TODO: need a failure type" type - CapArgs {.preservesDictionary.} = object + BootArgs {.preservesDictionary.} = object dataspace: Cap - ClientSideArgs {.preservesDictionary.} = object - `listen-socket`: string - DaemonSideArgs {.preservesDictionary.} = object - `daemon-socket`: string - -proc runNixActor(nixState: EvalState) = - let erisStore = newMemoryStore() - runActor("nix_actor") do (root: Cap; turn: var Turn): - connectStdio(root, turn) - - let pat = ?CapArgs - during(turn, root, pat) do (ds: Cap): - discard publish(turn, ds, - initRecord("nixVersion", toPreserve($nixVersion.c_str))) +proc main() = + initLibexpr() - discard bootNixFacet(turn, ds) - - let pat = ?Observe(pattern: !Eval) ?? {0: grabLit(), 1: grabDict()} - during(turn, ds, pat) do (e: string, o: Assertion): - var ass = Eval(expr: e) - doAssert fromPreserve(ass.options, unpackLiterals(o)) - # unused options + runActor("nix_actor") do (root: Cap; turn: var Turn): + connectStdio(turn, root) + + during(turn, root, ?BootArgs) do (ds: Cap): + let + store = openStore() + state = newState(store) + + let pat = ?Observe(pattern: !Eval) ?? {0: grabLit(), 1: grabLit()} + during(turn, ds, pat) do (expr: string, path: string): + var + value: Value + ass = Eval(expr: expr, path: path) try: - ass.result = eval(nixState, ass.expr) + value = evalFromString(state, ass.expr, ass.path) + force(state, value) + ass.result = toPreserve(state, value, void) discard publish(turn, ds, ass) except CatchableError as err: stderr.writeLine "failed to evaluate ", ass.expr, ": ", err.msg - except StdException as err: - stderr.writeLine "failed to evaluate ", ass.expr, ": ", err.what - - during(turn, root, ?ClientSideArgs) do (socketPath: string): - bootClientSide(turn, ds, erisStore, socketPath) - - during(turn, root, ?DaemonSideArgs) do (socketPath: string): - bootDaemonSide(turn, ds, erisStore, socketPath) - -proc main = - initNix() - initGC() - let nixStore = openStore() - runNixActor(newEvalState(nixStore)) + close(value) + do: + close(state) + close(store) main() diff --git a/src/nix_actor.nim.cfg b/src/nix_actor.nim.cfg deleted file mode 100644 index 1dcd32bde059..000000000000 --- a/src/nix_actor.nim.cfg +++ /dev/null @@ -1 +0,0 @@ -backend:cpp diff --git a/src/nix_actor/daemons.nim b/src/nix_actor/daemons.nim index 3bc5591744ed..8e3a7fbb0a4f 100644 --- a/src/nix_actor/daemons.nim +++ b/src/nix_actor/daemons.nim @@ -155,14 +155,11 @@ proc callDaemon(turn: var Turn; path: string; action: proc (daemon: Session; tur proc bootDaemonSide*(turn: var Turn; ds: Cap; store: ErisStore; socketPath: string) = - during(turn, ds, ?Observe(pattern: !Missing) ?? {0: grab()}) do (a: Preserve[Cap]): + during(turn, ds, ?Observe(pattern: !Missing) ?? {0: grab()}) do (targets: Literal[StringSeq]): # cannot use `grabLit` here because an array is a compound # TODO: unpack to a `Pattern` let daemon = callDaemon(turn, socketPath) do (daemon: Session; turn: var Turn): - var targets: StringSeq - doAssert targets.fromPreserve(unpackLiterals(a)) - # unpack ]> - let missFut = queryMissing(daemon, targets) + let missFut = queryMissing(daemon, targets.value) addCallback(missFut, turn) do (turn: var Turn): close(daemon) var miss = read(missFut) @@ -180,15 +177,12 @@ proc bootDaemonSide*(turn: var Turn; ds: Cap; store: ErisStore; socketPath: stri do: close(daemon) - during(turn, ds, ?Observe(pattern: !PathInfo) ?? {1: grabDict()}) do (pat: Value): - var daemon: Session - var request: AddToStoreClientAttrs - if request.fromPreserve(unpackLiterals pat): - daemon = callDaemon(turn, socketPath) do (daemon: Session; turn: var Turn): - let fut = addToStore(daemon, store, request) - addCallback(fut, turn) do (turn: var Turn): - close(daemon) - var (path, info) = read(fut) - discard publish(turn, ds, initRecord("path", path.toPreserve, info.toPreserve)) + during(turn, ds, ?Observe(pattern: !PathInfo) ?? {1: grabDict()}) do (request: Literal[AddToStoreClientAttrs]): + let daemon = callDaemon(turn, socketPath) do (daemon: Session; turn: var Turn): + let fut = addToStore(daemon, store, request.value) + addCallback(fut, turn) do (turn: var Turn): + close(daemon) + var (path, info) = read(fut) + discard publish(turn, ds, initRecord("path", path.toPreserve, info.toPreserve)) do: close(daemon) diff --git a/src/nix_actor/nix_api.nim b/src/nix_actor/nix_api.nim new file mode 100644 index 000000000000..2093773899ce --- /dev/null +++ b/src/nix_actor/nix_api.nim @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +import ./nix_api_types, ./nix_api_util, ./nix_api_value, ./nix_api_store, ./nix_api_expr + +export NixContext, Store, State, Value, ValueType, + gc_decref, isNil + +{.passC: staticExec("pkg-config --cflags nix-expr-c").} +{.passL: staticExec("pkg-config --libs nix-expr-c").} + +# Always pass NixContext as a nil pointer. + +proc initLibexpr* = + var ctx: NixContext + discard libexpr_init(ctx) + +proc openStore*(uri: string, params: varargs[string]): Store = + var ctx: NixContext + var args = allocCStringArray(params) + defer: deallocCStringArray(args) + result = store_open(ctx, uri, addr args) + +proc openStore*(): Store = + var ctx: NixContext + result = store_open(ctx, nil, nil) + +proc close*(store: Store) = store_unref(store) + +proc newState*(store: Store; searchPath: varargs[string]): State = + var ctx: NixContext + var path = allocCStringArray(searchPath) + defer: deallocCStringArray(path) + result = state_create(ctx, path, store) + +proc close*(state: State) = state_free(state) + +proc newValue*(state: State): Value = + var ctx: NixContext + alloc_value(ctx, state) + +proc evalFromString*(state: State; expr, path: string): Value = + var ctx: NixContext + result = alloc_value(ctx, state) + discard expr_eval_from_string(ctx, state, expr, path, result) + +proc close*(value: Value) = + var ctx: NixContext + discard gc_decref(ctx, cast[pointer](value)) + +proc force*(state: State; value: Value) = + var ctx: NixContext + discard value_force(ctx, state, value) + +proc get_attr_byidx*(ctx: NixContext; value: Value; state: State; i: cuint): (cstring, Value) = + var ctx: NixContext + result[1] = get_attr_byidx(ctx, value, state, i, addr result[0]) + diff --git a/src/nix_actor/nix_api_expr.nim b/src/nix_actor/nix_api_expr.nim new file mode 100644 index 000000000000..d3cd6cc983e5 --- /dev/null +++ b/src/nix_actor/nix_api_expr.nim @@ -0,0 +1,28 @@ +## Module generated by c2nim for nix_api_expr.h + +import ./nix_api_types + +{.pragma: nix_api_expr, header: "nix_api_expr.h", importc: "nix_$1".} + +proc libexpr_init*(context: NixContext): nix_err {.nix_api_expr.} + +proc expr_eval_from_string*(context: NixContext; state: State; expr: cstring; path: cstring; value: Value): nix_err {.nix_api_expr.} + +proc value_call*(context: NixContext; state: State; fn: Value; arg: Value; value: Value): nix_err {.nix_api_expr.} + +proc value_force*(context: NixContext; state: State; value: Value): nix_err {.nix_api_expr.} + +proc value_force_deep*(context: NixContext; state: State; value: Value): nix_err {.nix_api_expr.} + +proc state_create*(context: NixContext; searchPath: cstringArray; store: Store): State {.nix_api_expr.} + +proc state_free*(state: State) {.nix_api_expr.} + +proc gc_incref*(context: NixContext; `object`: pointer): nix_err {.nix_api_expr.} + +proc gc_decref*(context: NixContext; `object`: pointer): nix_err {.nix_api_expr.} + +proc gc_now*() {.nix_api_expr.} + +proc gc_register_finalizer*(obj: pointer; cd: pointer; finalizer: proc (obj: pointer; cd: pointer)) {.nix_api_expr.} + diff --git a/src/nix_actor/nix_api_store.nim b/src/nix_actor/nix_api_store.nim new file mode 100644 index 000000000000..6e5ccfee0b71 --- /dev/null +++ b/src/nix_actor/nix_api_store.nim @@ -0,0 +1,26 @@ +## Module generated by c2nim for nix_api_store.h + +import ./nix_api_types + +{.pragma: nix_api_store, header: "nix_api_store.h", importc: "nix_$1".} + +proc libstore_init*(context: NixContext): nix_err {.nix_api_store.} + +proc init_plugins*(context: NixContext): nix_err {.nix_api_store.} + +proc store_open*(a1: NixContext; uri: cstring; params: ptr cstringArray): Store {.nix_api_store.} + +proc store_unref*(store: Store) {.nix_api_store.} + +proc store_get_uri*(context: NixContext; store: Store; dest: cstring; n: cuint): nix_err {.nix_api_store.} + +proc store_parse_path*(context: NixContext; store: Store; path: cstring): StorePath {.nix_api_store.} + +proc store_path_free*(p: StorePath) {.nix_api_store.} + +proc store_is_valid_path*(context: NixContext; store: Store; path: StorePath): bool {.nix_api_store.} + +proc store_build*(context: NixContext; store: Store; path: StorePath; userdata: pointer; callback: proc (userdata: pointer; outname: cstring; `out`: cstring)): nix_err {.nix_api_store.} + +proc store_get_version*(a1: NixContext; store: Store; dest: cstring; n: cuint): nix_err {.nix_api_store.} + diff --git a/src/nix_actor/nix_api_util.nim b/src/nix_actor/nix_api_util.nim new file mode 100644 index 000000000000..2a78428707a4 --- /dev/null +++ b/src/nix_actor/nix_api_util.nim @@ -0,0 +1,35 @@ +## Module generated by c2nim for nix_api_util.h + +import ./nix_api_types + +{.pragma: nix_api_util, header: "nix_api_util.h", importc: "nix_$1".} +{.pragma: importUtil, header: "nix_api_util.h", importc.} + +var + NIX_OK* {.importUtil.}: cint + NIX_ERR_UNKNOWN* {.importUtil.}: cint + NIX_ERR_OVERFLOW* {.importUtil.}: cint + NIX_ERR_KEY* {.importUtil.}: cint + NIX_ERR_NIX_ERROR* {.importUtil.}: cint + +proc c_context_create*(): NixContext {.nix_api_util.} + +proc c_context_free*(context: NixContext) {.nix_api_util.} + +proc libutil_init*(context: NixContext): nix_err {.nix_api_util.} + +proc setting_get*(context: NixContext; key: cstring; value: cstring; n: cint): nix_err {.nix_api_util.} + +proc setting_set*(context: NixContext; key: cstring; value: cstring): nix_err {.nix_api_util.} + +proc version_get*(): cstring {.nix_api_util.} + +proc err_msg*(context: NixContext; ctx: NixContext; n: ptr cuint): cstring {.nix_api_util.} + +proc err_info_msg*(context: NixContext; read_context: NixContext; value: cstring; n: cint): nix_err {.nix_api_util.} + +proc err_name*(context: NixContext; read_context: NixContext; value: cstring; n: cint): nix_err {.nix_api_util.} + +proc err_code*(read_context: NixContext): nix_err {.nix_api_util.} + +proc set_err_msg*(context: NixContext; err: nix_err; msg: cstring): nix_err {.nix_api_util.} diff --git a/src/nix_actor/nix_api_value.nim b/src/nix_actor/nix_api_value.nim new file mode 100644 index 000000000000..057f6383a4fd --- /dev/null +++ b/src/nix_actor/nix_api_value.nim @@ -0,0 +1,72 @@ +## Module generated by c2nim for nix_api_value.h + +import ./nix_api_types + +type + PrimOpFun* = proc (user_data: pointer; context: NixContext; state: ptr State; args: ptr Value; ret: Value) + +# proc alloc_primop*(context: NixContext; fun: PrimOpFun; arity: cint; name: cstring; args: cstringArray; doc: cstring; user_data: pointer): ptr PrimOp {.importc: "nix_$1", header: "nix_api_value.h".} + +# proc register_primop*(context: NixContext; primOp: ptr PrimOp): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} + +proc alloc_value*(context: NixContext; state: State): Value {.importc: "nix_$1", header: "nix_api_value.h".} + +proc get_type*(context: NixContext; value: Value): ValueType {.importc: "nix_$1", header: "nix_api_value.h".} + +proc get_typename*(context: NixContext; value: Value): cstring {.importc: "nix_$1", header: "nix_api_value.h".} + +proc get_bool*(context: NixContext; value: Value): bool {.importc: "nix_$1", header: "nix_api_value.h".} + +proc get_string*(context: NixContext; value: Value): cstring {.importc: "nix_$1", header: "nix_api_value.h".} + +proc get_path_string*(context: NixContext; value: Value): cstring {.importc: "nix_$1", header: "nix_api_value.h".} + +proc get_list_size*(context: NixContext; value: Value): cuint {.importc: "nix_$1", header: "nix_api_value.h".} + +proc get_attrs_size*(context: NixContext; value: Value): cuint {.importc: "nix_$1", header: "nix_api_value.h".} + +proc get_float*(context: NixContext; value: Value): cdouble {.importc: "nix_$1", header: "nix_api_value.h".} + +proc get_int*(context: NixContext; value: Value): int64 {.importc: "nix_$1", header: "nix_api_value.h".} + +# proc get_external*(context: NixContext; a2: Value): ptr ExternalValue {.importc: "nix_$1", header: "nix_api_value.h".} + +proc get_list_byidx*(context: NixContext; value: Value; state: State; ix: cuint): Value {.importc: "nix_$1", header: "nix_api_value.h".} + +proc get_attr_byname*(context: NixContext; value: Value; state: State; name: cstring): Value {.importc: "nix_$1", header: "nix_api_value.h".} + +proc has_attr_byname*(context: NixContext; value: Value; state: State; name: cstring): bool {.importc: "nix_$1", header: "nix_api_value.h".} + +proc get_attr_byidx*(context: NixContext; value: Value; state: State; i: cuint; name: ptr cstring): Value {.importc: "nix_$1", header: "nix_api_value.h".} + +proc get_attr_name_byidx*(context: NixContext; value: Value; state: State; i: cuint): cstring {.importc: "nix_$1", header: "nix_api_value.h".} + +proc set_bool*(context: NixContext; value: Value; b: bool): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} + +proc set_string*(context: NixContext; value: Value; str: cstring): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} + +proc set_path_string*(context: NixContext; value: Value; str: cstring): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} + +proc set_float*(context: NixContext; value: Value; d: cdouble): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} + +proc set_int*(context: NixContext; value: Value; i: int64): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} + +proc set_null*(context: NixContext; value: Value): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} + +# proc set_external*(context: NixContext; value: Value; val: ptr ExternalValue): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} + +proc make_list*(context: NixContext; s: State; value: Value; size: cuint): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} + +proc set_list_byidx*(context: NixContext; value: Value; ix: cuint; elem: Value): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} + +# proc make_attrs*(context: NixContext; value: Value; b: ptr BindingsBuilder): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} + +# proc set_primop*(context: NixContext; value: Value; op: ptr PrimOp): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} + +proc copy_value*(context: NixContext; value: Value; source: Value): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} + +# proc make_bindings_builder*(context: NixContext; state: State; capacity: csize_t): ptr BindingsBuilder {.importc: "nix_$1", header: "nix_api_value.h".} + +# proc bindings_builder_insert*(context: NixContext; builder: ptr BindingsBuilder; name: cstring; value: Value): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} + +# proc bindings_builder_free*(builder: ptr BindingsBuilder) {.importc: "nix_$1", header: "nix_api_value.h".} diff --git a/src/nix_actor/old.nim b/src/nix_actor/old.nim new file mode 100644 index 000000000000..78ef76237be9 --- /dev/null +++ b/src/nix_actor/old.nim @@ -0,0 +1,369 @@ +type + Snoop = ref object + client, daemon: AsyncSocket + buffer: seq[Word] + version: Version + +type ValidPathInfo = object + path: string + deriver: string + narHash: string + references: StringSet + registrationTime, narSize: BiggestInt + ultimate: bool + sigs: StringSet + ca: string + +proc send(session: Snoop; sock: AsyncSocket; words: varargs[Word]): Future[void] = + for i, word in words: session.buffer[i] = word + send(sock, addr session.buffer[0], words.len shl 3) + +proc send(session: Snoop; sock: AsyncSocket; s: string): Future[void] = + let wordCount = (s.len + 7) shr 3 + if wordCount > session.buffer.len: setLen(session.buffer, wordCount) + session.buffer[0] = Word s.len + if wordCount > 0: + session.buffer[wordCount] = 0x00 + copyMem(addr session.buffer[1], unsafeAddr s[0], s.len) + send(sock, addr session.buffer[0], (succ wordCount) shl 3) + +proc passWord(a, b: AsyncSocket): Future[Word] {.async.} = + var w = await recvWord(a) + await send(b, addr w, sizeof(Word)) + return w + +proc passString(session: Snoop; a, b: AsyncSocket): Future[string] {.async.} = + var s = await recvString(a) + await send(session, b, s) + return s + +proc passStringSeq(session: Snoop; a, b: AsyncSocket): Future[seq[string]] {.async.} = + let count = int(await passWord(a, b)) + var strings = newSeq[string](count) + for i in 0..= 16 + info.ultimate = (await passDaemonWord(session)) != 0 + info.sigs = await passDaemonStringSet(session) + info.ca = await passDaemonString(session) + return info + +proc passChunks(session: Snoop; a, b: AsyncSocket): Future[int] {.async.} = + var total: int + while true: + let chunkLen = int(await passWord(a, b)) + if chunkLen == 0: + break + else: + let wordLen = (chunkLen + 7) shr 3 + if session.buffer.len < wordLen: setLen(session.buffer, wordLen) + let recvLen = await recvInto(a, addr session.buffer[0], chunkLen) + # each chunk must be recved contiguously + if recvLen != chunkLen: + raise newException(ProtocolError, "invalid chunk read") + await send(b, addr session.buffer[0], recvLen) + inc(total, recvLen) + return total + +proc passClientChunks(session: Snoop): Future[int] = + passChunks(session, session.client, session.daemon) + +proc passErrorDaemonError(session: Snoop) {.async.} = + let + typ = await passDaemonString(session) + assert typ == "Error" + let + lvl = await passDaemonWord(session) + name = await passDaemonString(session) + msg = passDaemonString(session) + havePos = await passDaemonWord(session) + assert havePos == 0 + let + nrTraces = await passDaemonWord(session) + for i in 1..nrTraces: + let havPos = await passDaemonWord(session) + assert havPos == 0 + let msg = await passDaemonString(session) + +proc passDaemonFields(session: Snoop): Future[Fields] {.async.} = + let count = await passDaemonWord(session) + var fields = newSeq[Field](count) + for i in 0..= 26 + await passErrorDaemonError(session) + + of STDERR_NEXT: + let s = await passDaemonString(session) + + of STDERR_START_ACTIVITY: + var act: ActionStart + act.id = BiggestInt(await passDaemonWord(session)) + act.level = BiggestInt(await passDaemonWord(session)) + act.`type` = BiggestInt(await passDaemonWord(session)) + act.text = await passDaemonString(session) + act.fields = await passDaemonFields(session) + act.parent = BiggestInt(await passDaemonWord(session)) + + of STDERR_STOP_ACTIVITY: + var act: ActionStop + act.id = BiggestInt(await passDaemonWord(session)) + + of STDERR_RESULT: + var act: ActionResult + act.id = BiggestInt(await passDaemonWord(session)) + act.`type` = BiggestInt(await passDaemonWord(session)) + act.fields = await passDaemonFields(session) + + of STDERR_LAST: + break + + else: + raise newException(ProtocolError, "unknown work verb " & $word) + +#[ +proc fromClient(miss: var Missing; socket: AsyncSocket) {.async.} = + result.targets = await passClientStringSet(session) + +proc fromDaemon(miss: var Missing; socket: AsyncSocket) {.async.} = + miss.willBuild = await passDaemonStringSet(session) + miss.willSubstitute = await passDaemonStringSet(session) + miss.unknown = await passDaemonStringSet(session) + miss.downloadSize = BiggestInt await passDaemonWord(session) + miss.narSize = BiggestInt await passDaemonWord(session) +]# + +proc loop(session: Snoop) {.async.} = + var chunksTotal: int + try: + while not session.client.isClosed: + let wop = await passClientWord(session) + case wop + of wopIsValidPath: + let path = await passClientString(session) + stderr.writeLine "wopIsValidPath ", path + await passWork(session) + let word = await passDaemonWord(session) + + of wopAddToStore: + assert session.version.minor >= 25 + let + name = await passClientString(session) + caMethod = await passClientString(session) + refs = await passClientStringSet(session) + repairBool = await passClientWord(session) + stderr.writeLine "wopAddToStore ", name + let n = await passClientChunks(session) + inc(chunksTotal, n) + await passWork(session) + let info = await passDaemonValidPathInfo(session, true) + + of wopAddTempRoot: + let path = await passClientString(session) + stderr.writeLine "wopAddTempRoot ", path + await passWork(session) + discard await passDaemonWord(session) + + of wopAddIndirectRoot: + let path = await passClientString(session) + stderr.writeLine "wopAddIndirectRoot ", path + await passWork(session) + discard await passDaemonWord(session) + + of wopSetOptions: + discard passClientWord(session) # keepFailed + discard passClientWord(session) # keepGoing + discard passClientWord(session) # tryFallback + discard passClientWord(session) # verbosity + discard passClientWord(session) # maxBuildJobs + discard passClientWord(session) # maxSilentTime + discard passClientWord(session) # useBuildHook + discard passClientWord(session) # verboseBuild + discard passClientWord(session) # logType + discard passClientWord(session) # printBuildTrace + discard passClientWord(session) # buildCores + discard passClientWord(session) # useSubstitutes + assert session.version.minor >= 12 + let overrides = await passClientStringMap(session) + await passWork(session) + + of wopQueryPathInfo: + assert session.version >= 17 + let path = await passClientString(session) + stderr.writeLine "wopQueryPathInfo ", path + await passWork(session) + let valid = await passDaemonWord(session) + if valid != 0: + var info = await passDaemonValidPathInfo(session, false) + info.path = path + stderr.writeLine "wopQueryPathInfo ", $info + + of wopQueryMissing: + assert session.version >= 30 + var miss: Missing + miss.targets = await passClientStringSeq(session) + await passWork(session) + miss.willBuild = await passDaemonStringSet(session) + miss.willSubstitute = await passDaemonStringSet(session) + miss.unknown = await passDaemonStringSet(session) + miss.downloadSize = BiggestInt await passDaemonWord(session) + miss.narSize = BiggestInt await passDaemonWord(session) + stderr.writeLine "wopQueryMissing ", $miss + + of wopBuildPathsWithResults: + assert session.version >= 34 + let + drvs = await passClientStringSeq(session) + buildMode = await passClientWord(session) + stderr.writeLine "wopBuildPathsWithResults drvs ", $drvs + await passWork(session) + let count = await passDaemonWord(session) + for _ in 1..count: + let + path = await passDaemonString(session) + status = await passDaemonWord(session) + errorMsg = await passDaemonString(session) + timesBUild = await passDaemonWord(session) + isNonDeterministic = await passDaemonWord(session) + startTime = await passDaemonWord(session) + stopTime = await passDaemonWord(session) + outputs = await passDaemonStringMap(session) + + else: + stderr.writeLine "unknown worker op ", wop.int + break + except ProtocolError as err: + stderr.writeLine "connection terminated" + stderr.writeLine "chunk bytes transfered: ", formatSize(chunksTotal) + finally: + close(session.daemon) + close(session.client) + +proc handshake(listener: AsyncSocket): Future[Snoop] {.async.} = + ## Take the next connection from `listener` and return a `Session`. + let session = Snoop(buffer: newSeq[Word](1024)) # 8KiB + session.client = await listener.accept() + session.daemon = newAsyncSocket( + domain = AF_UNIX, + sockType = SOCK_STREAM, + protocol = cast[Protocol](0), + buffered = false, + ) + await connectUnix(session.daemon, daemonSocketPath()) + let clientMagic = await passClientWord(session) + if clientMagic != WORKER_MAGIC_1: + raise newException(ProtocolError, "invalid protocol magic") + let daemonMagic = await passDaemonWord(session) + let daemonVersion = await passDaemonWord(session) + session.version = Version(await passClientWord(session)) + if session.version < PROTOCOL_VERSION: + raise newException(ProtocolError, "obsolete protocol version") + assert session.version.minor >= 14 + discard await(passClientWord(session)) + # obsolete CPU affinity + assert session.version.minor >= 11 + discard await(passClientWord(session)) + # obsolete reserveSpace + assert session.version.minor >= 33 + let daemonVersionString = await passDaemonString(session) + assert daemonVersionString == $store.nixVersion + await passWork(session) + return session + +proc emulateSocket*(path: string) {.async, gcsafe.} = + let listener = newAsyncSocket( + domain = AF_UNIX, + sockType = SOCK_STREAM, + protocol = cast[Protocol](0), + buffered = false) + bindUnix(listener, path) + listen(listener) + stderr.writeLine "listening on ", path + while not listener.isClosed: + try: + let session = await handshake(listener) + assert not session.isNil + asyncCheck loop(session) + except ProtocolError: + close(session) + finally: + close(session) diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 6f316278dd27..4a485f7b42b2 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -5,7 +5,7 @@ import type Eval* {.preservesRecord: "eval".} = object `expr`*: string - `options`*: Table[Symbol, Preserve[void]] + `path`*: string `result`*: Preserve[void] AttrSet* = Table[Symbol, Preserve[void]] -- cgit 1.4.1 From c59bd0fc342e046641e3fef3417947a4a775905c Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sat, 22 Jun 2024 16:42:50 +0300 Subject: Replace protocol, update dependencies --- README.md | 85 +++--- Tupfile | 6 +- Tuprules.jq | 12 + Tuprules.tup | 5 +- build-nim-sbom.nix | 196 +++++++++++++ default.nix | 14 + lock.json | 171 ------------ nix_actor.nimble | 69 ++++- protocol.prs | 98 +++---- sbom.json | 596 ++++++++++++++++++++++++++++++++++++++++ shell.nix | 17 -- src/Tupfile | 3 - src/nix_actor.nim | 209 +++++++++++--- src/nix_actor/Tupfile | 2 +- src/nix_actor/clients.nim | 174 ------------ src/nix_actor/daemons.nim | 188 ------------- src/nix_actor/nix_api.nim | 21 +- src/nix_actor/nix_api_expr.nim | 17 +- src/nix_actor/nix_api_store.nim | 2 +- src/nix_actor/nix_api_types.nim | 25 ++ src/nix_actor/nix_api_value.nim | 89 +++--- src/nix_actor/protocol.nim | 190 ++++++------- src/nix_actor/sockets.nim | 213 -------------- 23 files changed, 1314 insertions(+), 1088 deletions(-) create mode 100755 Tuprules.jq create mode 100644 build-nim-sbom.nix create mode 100644 default.nix delete mode 100644 lock.json create mode 100644 sbom.json delete mode 100644 shell.nix delete mode 100644 src/Tupfile delete mode 100644 src/nix_actor/clients.nim delete mode 100644 src/nix_actor/daemons.nim create mode 100644 src/nix_actor/nix_api_types.nim delete mode 100644 src/nix_actor/sockets.nim diff --git a/README.md b/README.md index 1d07f44efbf8..d3b7b753672c 100644 --- a/README.md +++ b/README.md @@ -4,59 +4,62 @@ An actor for interacting with the [Nix](https://nixos.org/) daemon via the [Synd See [protocol.prs](./protocol.prs) for the Syndicate protocol [schema](https://preserves.dev/preserves-schema.html). -*This is only a proof-of-concept and is not yet useful.* +The was once an abstraction of the Nix worker socket that could intermediate between clients and the worker but that code has been removed, refer to git history for that. + +*This is only a proof-of-concept and is probably not useful yet.* ## Example configuration A demo script for the [Syndicate server](https://git.syndicate-lang.org/syndicate-lang/syndicate-rs), see https://synit.org/book/operation/scripting.html ``` -? $nixspace [ +#!/usr/bin/env -S syndicate-server --config - ? {}; in pkgs.hello" { } ?drv> [ - ? [ - $log ! - ] - ] +let ?nixLog = dataspace +$nixLog ?? ?line [ + # Log out to the syndicate-server log. + $log ! +] - ? [ - $log ! - ] +let ?results = dataspace +$results ? ?any [ + # Just log everything else. + $log ! +] - ? [ - $log ! - ] +let ?resolver = dataspace +$resolver ? $cap [ - ? [ - $log ! - ] + - ? [ - $log ! + {}; in pkgs.cowsay" $nixLog $results> + $results ? [ + $log ! + $cap ] - $config [ - > - ? ?cap> [ - $cap { - dataspace: $nixspace - daemon-socket: "/nix/var/nix/daemon-socket/socket" - listen-socket: "/tmp/translator.worker.nix.socket" - } - ] - - ] ] + +? ?cap> [ + $cap $resolver> +] + +> + + ``` diff --git a/Tupfile b/Tupfile index 28c450e9095e..ba7180cce451 100644 --- a/Tupfile +++ b/Tupfile @@ -1,2 +1,6 @@ include_rules -: lock.json |> !nim_cfg |> | ./ + +: sbom.json |> !sbom-to-nix |> | ./ +run ./Tuprules.jq sbom.json + +: foreach {bin} |> !assert_built |> diff --git a/Tuprules.jq b/Tuprules.jq new file mode 100755 index 000000000000..2c9a5e70e9f5 --- /dev/null +++ b/Tuprules.jq @@ -0,0 +1,12 @@ +#! /usr/bin/env -S jq --raw-output --from-file +.metadata.component.properties as $props | +$props | + ( map( select(.name | .[0:10] == "nim:binDir") ) + + map( select(.name | .[0:10] == "nim:srcDir") ) | + map( .value ) + ) + ["."] | .[0] as $binDir | + +$props | + map( select(.name | .[0:8] == "nim:bin:") ) | + map( ": \($binDir)/\(.value).nim |> !nim_bin |> $(BIN_DIR)/\(.name[8:]) {bin}" ) | + join("\n") diff --git a/Tuprules.tup b/Tuprules.tup index 0d08158a620e..1b889871461d 100644 --- a/Tuprules.tup +++ b/Tuprules.tup @@ -1,7 +1,8 @@ -include ../eris-nim/depends.tup -NIM_FLAGS += --path:$(TUP_CWD)/../eris-nim/src +PROJECT_DIR = $(TUP_CWD) +NIM = $(DIRENV) $(NIM) include ../syndicate-nim/depends.tup NIM_FLAGS += --path:$(TUP_CWD)/../syndicate-nim/src NIM_GROUPS += $(TUP_CWD)/ +NIM_GROUPS += $(TUP_CWD)/ diff --git a/build-nim-sbom.nix b/build-nim-sbom.nix new file mode 100644 index 000000000000..f1db25be7072 --- /dev/null +++ b/build-nim-sbom.nix @@ -0,0 +1,196 @@ +{ + lib, + stdenv, + fetchgit, + fetchzip, + runCommand, + xorg, + nim, + nimOverrides, +}: + +let + fetchers = { + fetchzip = + { url, sha256, ... }: + fetchzip { + name = "source"; + inherit url sha256; + }; + fetchgit = + { + fetchSubmodules ? false, + leaveDotGit ? false, + rev, + sha256, + url, + ... + }: + fetchgit { + inherit + fetchSubmodules + leaveDotGit + rev + sha256 + url + ; + }; + }; + + filterPropertiesToAttrs = + prefix: properties: + lib.pipe properties [ + (builtins.filter ({ name, ... }: (lib.strings.hasPrefix prefix name))) + (map ( + { name, value }: + { + name = lib.strings.removePrefix prefix name; + inherit value; + } + )) + builtins.listToAttrs + ]; + + buildNimCfg = + { backend, components, ... }: + let + componentSrcDirs = map ( + { properties, ... }: + let + fodProps = filterPropertiesToAttrs "nix:fod:" properties; + fod = fetchers.${fodProps.method} fodProps; + srcDir = fodProps.srcDir or ""; + in + if srcDir == "" then fod else "${fod}/${srcDir}" + ) components; + in + runCommand "nim.cfg" + { + outputs = [ + "out" + "src" + ]; + nativeBuildInputs = [ xorg.lndir ]; + } + '' + pkgDir=$src/pkg + cat << EOF >> $out + backend:${backend} + path:"$src" + path:"$pkgDir" + EOF + mkdir -p "$pkgDir" + ${lib.strings.concatMapStrings (d: '' + lndir "${d}" "$pkgDir" + '') componentSrcDirs} + ''; + + buildCommands = lib.attrsets.mapAttrsToList ( + output: input: '' + nim compile $nimFlags --out:${output} ${input} + '' + ); + + installCommands = lib.attrsets.mapAttrsToList ( + output: input: '' + install -Dt $out/bin ${output} + '' + ); +in + +callerArg: sbomArg: + +let + applySbom = + { + passthru ? { }, + ... + }@prevAttrs: + let + sbom = lib.attrsets.recursiveUpdate ( + if builtins.isAttrs sbomArg then sbomArg else builtins.fromJSON (builtins.readFile sbomArg) + ) passthru.sbom or { }; + + properties = # SBOM metadata.component.properties as an attrset. + lib.attrsets.recursiveUpdate (builtins.listToAttrs sbom.metadata.component.properties) + passthru.properties or { }; + + nimBin = # A mapping of Nim module file paths to names of programs. + lib.attrsets.recursiveUpdate (lib.pipe properties [ + (lib.attrsets.filterAttrs (name: value: lib.strings.hasPrefix "nim:bin:" name)) + (lib.attrsets.mapAttrs' ( + name: value: { + name = lib.strings.removePrefix "nim:bin:" name; + value = "${properties."nim:binDir" or (properties."nim:srcDir" or ".")}/${value}"; + } + )) + ]) passthru.nimBin or { }; + in + { + strictDeps = true; + + pname = prevAttrs.pname or sbom.metadata.component.name; + version = prevAttrs.version or sbom.metadata.component.version or null; + + configurePhase = + prevAttrs.configurePhase or '' + runHook preConfigure + echo "nim.cfg << $nimCfg" + cat $nimCfg >> nim.cfg + cat << EOF >> nim.cfg + nimcache:"$NIX_BUILD_TOP/nimcache" + parallelBuild:$NIX_BUILD_CORES + EOF + runHook postConfigure + ''; + + buildPhase = + prevAttrs.buildPhase or '' + runHook preBuild + ${lib.strings.concatLines (buildCommands nimBin)} + runHook postBuild + ''; + + installPhase = + prevAttrs.installPhase or '' + runHook preInstall + ${lib.strings.concatLines (installCommands nimBin)} + runHook postInstall + ''; + + nativeBuildInputs = (prevAttrs.nativeBuildInputs or [ ]) ++ [ nim ]; + + nimCfg = + prevAttrs.nimCfg or (buildNimCfg { + backend = prevAttrs.nimBackend or properties."nim:backend" or "c"; + inherit (sbom) components; + }); + + passthru = { + inherit sbom properties nimBin; + }; + }; + + applyOverrides = + prevAttrs: + builtins.foldl' ( + prevAttrs: + { name, ... }@component: + if (builtins.hasAttr name nimOverrides) then + prevAttrs // (nimOverrides.${name} component prevAttrs) + else + prevAttrs + ) prevAttrs prevAttrs.passthru.sbom.components; + + composition = + finalAttrs: + let + callerAttrs = if builtins.isAttrs callerArg then callerArg else callerArg finalAttrs; + sbomAttrs = callerAttrs // (applySbom callerAttrs); + overrideAttrs = sbomAttrs // (applyOverrides sbomAttrs); + in + overrideAttrs; +in +stdenv.mkDerivation composition + +# TODO: Add an overrideSbom function into the result.. diff --git a/default.nix b/default.nix new file mode 100644 index 000000000000..db1dec61e79c --- /dev/null +++ b/default.nix @@ -0,0 +1,14 @@ +{ + pkgs ? import { }, +}: + +let + inherit (pkgs) lib; + buildNimSbom = pkgs.callPackage ./build-nim-sbom.nix { }; +in +buildNimSbom (finalAttrs: { + name = "nix-actor"; + nativeBuildInputs = [ pkgs.pkg-config ]; + buildInputs = [ pkgs.nixVersions.latest ]; + src = if lib.inNixShell then null else lib.cleanSource ./.; +}) ./sbom.json diff --git a/lock.json b/lock.json deleted file mode 100644 index 08f588a19423..000000000000 --- a/lock.json +++ /dev/null @@ -1,171 +0,0 @@ -{ - "depends": [ - { - "method": "fetchzip", - "packages": [ - "base32" - ], - "path": "/nix/store/qcnchjsak3hyn4c6r0zd6qvm7j8y1747-source", - "ref": "0.1.3", - "rev": "f541038fbe49fdb118cc2002d29824b9fc4bfd61", - "sha256": "16gh1ifp9hslsg0is0v1ya7rxqfhq5hjqzc3pfdqvcgibp5ybh06", - "srcDir": "", - "url": "https://github.com/OpenSystemsLab/base32.nim/archive/f541038fbe49fdb118cc2002d29824b9fc4bfd61.tar.gz" - }, - { - "method": "fetchzip", - "packages": [ - "cbor" - ], - "path": "/nix/store/70cqa9s36dqnmsf179cn9psj77jhqi1l-source", - "ref": "20230619", - "rev": "a4a1affd45ba90bea24e08733ae2bd02fe058166", - "sha256": "005ib6im97x9pdbg6p0fy58zpdwdbkpmilxa8nhrrb1hnpjzz90p", - "srcDir": "src", - "url": "https://git.sr.ht/~ehmry/nim_cbor/archive/a4a1affd45ba90bea24e08733ae2bd02fe058166.tar.gz" - }, - { - "method": "fetchzip", - "packages": [ - "coap" - ], - "path": "/nix/store/pqj933cnw7r7hp46jrpjlwh1yr0jvckp-source", - "ref": "20230331", - "rev": "a134213b51a8d250684f2ba26802ffa97fae4ffb", - "sha256": "1wbix6d8l26nj7m3xinh4m2f27n4ma0yzs3x5lpann2ha0y51k8b", - "srcDir": "src", - "url": "https://codeberg.org/eris/nim-coap/archive/a134213b51a8d250684f2ba26802ffa97fae4ffb.tar.gz" - }, - { - "method": "fetchzip", - "packages": [ - "configparser" - ], - "path": "/nix/store/4zl5v7i6cj3f9sayvsjcx2h20lqwr9a6-source", - "ref": "newSection", - "rev": "695f1285d63f1954c25eb1f42798d90fa7bcbe14", - "sha256": "0b0pb5i0kir130ia2zf8zcgdz8awms161i6p83ri3nbgibbjnr37", - "srcDir": "src", - "url": "https://github.com/ehmry/nim-configparser/archive/695f1285d63f1954c25eb1f42798d90fa7bcbe14.tar.gz" - }, - { - "method": "fetchzip", - "packages": [ - "eris" - ], - "path": "/nix/store/lxa6ba8r9hhs06k6f2iyznwjxix1klv1-source", - "ref": "20230823", - "rev": "49d8117367d3530533dc1d6a9111ddd134b08b1e", - "sha256": "0lq9a04cayf04nnhn0gvp5phlij0cis38v7cz7jmgks2xvz1bcbr", - "srcDir": "src", - "url": "https://codeberg.org/eris/nim-eris/archive/49d8117367d3530533dc1d6a9111ddd134b08b1e.tar.gz" - }, - { - "method": "fetchzip", - "packages": [ - "freedesktop_org" - ], - "path": "/nix/store/98wncmx58cfnhv3y96lzwm22zvyk9b1h-source", - "ref": "20230210", - "rev": "fb04d0862aca4be2edcc0eafa94b1840030231c8", - "sha256": "0wj5m09x1pr36gv8p5r72p6l3wwl01y8scpnlzx7q0h5ij6jaj6s", - "srcDir": "src", - "url": "https://git.sr.ht/~ehmry/freedesktop_org/archive/fb04d0862aca4be2edcc0eafa94b1840030231c8.tar.gz" - }, - { - "method": "fetchzip", - "packages": [ - "getdns" - ], - "path": "/nix/store/x9xmn7w4k6jg8nv5bnx148ibhnsfh362-source", - "ref": "20221222", - "rev": "c73cbe288d9f9480586b8fa87f6d794ffb6a6ce6", - "sha256": "1sbgx2x51szr22i72n7c8jglnfmr8m7y7ga0v85d58fwadiv7g6b", - "srcDir": "src", - "url": "https://git.sr.ht/~ehmry/getdns-nim/archive/c73cbe288d9f9480586b8fa87f6d794ffb6a6ce6.tar.gz" - }, - { - "method": "fetchzip", - "packages": [ - "hashlib" - ], - "path": "/nix/store/fav82xdbicvlk34nmcbl89zx99lr3mbs-source", - "rev": "f9455d4be988e14e3dc7933eb7cc7d7c4820b7ac", - "sha256": "1sx6j952lj98629qfgr7ds5aipyw9d6lldcnnqs205wpj4pkcjb3", - "srcDir": "", - "url": "https://github.com/ehmry/hashlib/archive/f9455d4be988e14e3dc7933eb7cc7d7c4820b7ac.tar.gz" - }, - { - "method": "fetchzip", - "packages": [ - "nimcrypto" - ], - "path": "/nix/store/zyr8zwh7vaiycn1s4r8cxwc71f2k5l0h-source", - "ref": "traditional-api", - "rev": "602c5d20c69c76137201b5d41f788f72afb95aa8", - "sha256": "1dmdmgb6b9m5f8dyxk781nnd61dsk3hdxqks7idk9ncnpj9fng65", - "srcDir": "", - "url": "https://github.com/cheatfate/nimcrypto/archive/602c5d20c69c76137201b5d41f788f72afb95aa8.tar.gz" - }, - { - "method": "fetchzip", - "packages": [ - "npeg" - ], - "path": "/nix/store/ffkxmjmigfs7zhhiiqm0iw2c34smyciy-source", - "ref": "1.2.1", - "rev": "26d62fdc40feb84c6533956dc11d5ee9ea9b6c09", - "sha256": "0xpzifjkfp49w76qmaylan8q181bs45anmp46l4bwr3lkrr7bpwh", - "srcDir": "src", - "url": "https://github.com/zevv/npeg/archive/26d62fdc40feb84c6533956dc11d5ee9ea9b6c09.tar.gz" - }, - { - "method": "fetchzip", - "packages": [ - "preserves" - ], - "path": "/nix/store/fmb2yckksz7iv3qdkk5gk1j060kppkq9-source", - "ref": "20231102", - "rev": "4faeb766dc3945bcfacaa1a836ef6ab29b20ceb0", - "sha256": "1a3g5bk1l1h250q3p6sqv6r1lpsplp330qqyp48r0i4a5r0jksq3", - "srcDir": "src", - "url": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/4faeb766dc3945bcfacaa1a836ef6ab29b20ceb0.tar.gz" - }, - { - "method": "fetchzip", - "packages": [ - "syndicate" - ], - "path": "/nix/store/nhpvl223vbzdrlzikw7pgyfxs344w7ma-source", - "ref": "20231108", - "rev": "095418032180e360ea27ec7fcd63193944b68e2c", - "sha256": "09pbml2chzz0v5zpz67fs7raj0mfmg8qrih2vz85xxc51h7ncqvw", - "srcDir": "src", - "url": "https://git.syndicate-lang.org/ehmry/syndicate-nim/archive/095418032180e360ea27ec7fcd63193944b68e2c.tar.gz" - }, - { - "method": "fetchzip", - "packages": [ - "taps" - ], - "path": "/nix/store/did1li0xk9qih80pvxqhjc4np3ijlfjj-source", - "ref": "20230331", - "rev": "4f9c9972d74eb39c662b43ed79d761e109bf00f1", - "sha256": "12qsizmisr1q0q4x37c5q6gmnqb5mp0bid7s3jlcsjvhc4jw2q57", - "srcDir": "src", - "url": "https://git.sr.ht/~ehmry/nim_taps/archive/4f9c9972d74eb39c662b43ed79d761e109bf00f1.tar.gz" - }, - { - "method": "fetchzip", - "packages": [ - "tkrzw" - ], - "path": "/nix/store/4x9wxyli4dy719svg1zaww0c0b3xckp0-source", - "ref": "20220922", - "rev": "efd87edb7b063182c1a1fa018006a87b515d589b", - "sha256": "1h0sdvai4gkkz48xfh67wa1xz2k8bkkba8q6snnbllmhmywd9apb", - "srcDir": "src", - "url": "https://git.sr.ht/~ehmry/nim-tkrzw/archive/efd87edb7b063182c1a1fa018006a87b515d589b.tar.gz" - } - ] -} diff --git a/nix_actor.nimble b/nix_actor.nimble index ad684e5e370a..21ac773171f3 100644 --- a/nix_actor.nimble +++ b/nix_actor.nimble @@ -1,8 +1,61 @@ -version = "20231208" -author = "Emery Hemingway" -description = "Syndicated Nix Actor" -license = "Unlicense" -srcDir = "src" -bin = @["nix_actor"] - -requires "nim >= 1.6.10", "syndicate >= 20231005", "eris >= 20230823" +# Emulate Nimble from CycloneDX data at sbom.json. + +import std/json + +proc lookupComponent(sbom: JsonNode; bomRef: string): JsonNode = + for c in sbom{"components"}.getElems.items: + if c{"bom-ref"}.getStr == bomRef: + return c + result = newJNull() + +let + sbom = (getPkgDir() & "/sbom.json").readFile.parseJson + comp = sbom{"metadata", "component"} + bomRef = comp{"bom-ref"}.getStr + +version = comp{"version"}.getStr +author = comp{"authors"}[0]{"name"}.getStr +description = comp{"description"}.getStr +license = comp{"licenses"}[0]{"license", "id"}.getStr + +for prop in comp{"properties"}.getElems.items: + let (key, val) = (prop{"name"}.getStr, prop{"value"}.getStr) + case key + of "nim:skipDirs:": + add(skipDirs, val) + of "nim:skipFiles:": + add(skipFiles, val) + of "nim:skipExt": + add(skipExt, val) + of "nim:installDirs": + add(installDirs, val) + of "nim:installFiles": + add(installFiles, val) + of "nim:installExt": + add(installExt, val) + of "nim:binDir": + add(binDir, val) + of "nim:srcDir": + add(srcDir, val) + of "nim:backend": + add(backend, val) + else: + if key.startsWith "nim:bin:": + namedBin[key[8..key.high]] = val + +for depend in sbom{"dependencies"}.items: + if depend{"ref"}.getStr == bomRef: + for depRef in depend{"dependsOn"}.items: + let dep = sbom.lookupComponent(depRef.getStr) + var spec = dep{"name"}.getStr + for extRef in dep{"externalReferences"}.elems: + if extRef{"type"}.getStr == "vcs": + spec = extRef{"url"}.getStr + break + let ver = dep{"version"}.getStr + if ver != "": + if ver.allCharsInSet {'0'..'9', '.'}: spec.add " == " + else: spec.add '#' + spec.add ver + requires spec + break diff --git a/protocol.prs b/protocol.prs index 2502aed92a0b..88bd54c3a42c 100644 --- a/protocol.prs +++ b/protocol.prs @@ -1,64 +1,44 @@ version 1 . -StringSeq = [string ...] . -StringSet = #{string} . -AttrSet = {symbol: any ...:...} . - -Build = . - -Realise = . - -Instantiate = . - -Eval = . - -Narinfo = . - -Field = int / string . -Fields = [Field ...] . - -ActionStart = . -ActionStop = . -ActionResult = . - -; TODO: why not make target a singleton? -Missing = . - -; Path info for the worker protocol version 35. -LegacyPathAttrs = { - deriver: string - narHash: string - references: StringSeq ; prefer a set - registrationTime: int - narSize: int - ultimate: bool - sigs: StringSet - ca: string -} . - -AddToStoreClientAttrs = { - name: string - eris: bytes - ca-method: symbol - references: StringSeq ; prefer a set +# Gatekeeper step to access nix-actor. +ResolveStep = . +ResolveDetail = { + # PATH to search for Nix utilities. + command-path: [string ...] + + # Command line options. + options: AttrSet + + # List of strings corresponding to entries in NIX_PATH. + # For example: + # [ "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos" + # "nixos-config=/etc/nixos/configuration.nix" + # "/nix/var/nix/profiles/per-user/root/channels" + # ] + lookupPath: [string ...] } . -; Intersection of the attributes needed to add a path to a store -; and the attributes returned by the daemon after adding the path. -AddToStoreAttrs = { - name: string - eris: bytes - ca-method: symbol - references: StringSeq ; prefer a set +# Common error type. +Error = . + +# Asserted to nix-actor. +# @expr is evaluated and asserted to @result +# with log lines messaged to @log. +Eval = . +EvalResult = Error / EvalSuccess . +EvalSuccess = [@expr string @result any] . + +# Asserted to nix-actor. +# @expr is instantiated to a store-derivation which is asserted to @result. +Instantiate = . +InstantiateResult = Error / Derivation . +Derivation = . + +# Asserted to nix-actor. +# The list of store-paths from realising @drv +# are asserted to @outputs as a list of strings. +Realise = . +RealiseResult = Error / Outputs . +Outputs = . - deriver: string - narHash: string - registrationTime: int - narSize: int - ultimate: bool - sigs: StringSet - ca: string -} . - -; Any collection of attributes describing a store path. -PathInfo = . +AttrSet = {symbol: any ...:...} . diff --git a/sbom.json b/sbom.json new file mode 100644 index 000000000000..f503d24f36f8 --- /dev/null +++ b/sbom.json @@ -0,0 +1,596 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "metadata": { + "component": { + "type": "application", + "bom-ref": "pkg:nim/nix_actor", + "name": "nix_actor", + "description": "Syndicated Nix Actor", + "version": "20240624", + "authors": [ + { + "name": "Emery Hemingway" + } + ], + "licenses": [ + { + "license": { + "id": "Unlicense" + } + } + ], + "properties": [ + { + "name": "nim:skipExt", + "value": "nim" + }, + { + "name": "nim:bin:nix-actor", + "value": "nix_actor" + }, + { + "name": "nim:srcDir", + "value": "src" + }, + { + "name": "nim:backend", + "value": "c" + } + ] + } + }, + "components": [ + { + "type": "library", + "bom-ref": "pkg:nim/syndicate", + "name": "syndicate", + "version": "20240623", + "externalReferences": [ + { + "url": "https://git.syndicate-lang.org/ehmry/syndicate-nim/archive/cdc59b76415730b46107a842b79c31d3ecbf48ea.tar.gz", + "type": "source-distribution" + }, + { + "url": "https://git.syndicate-lang.org/ehmry/syndicate-nim", + "type": "vcs" + } + ], + "properties": [ + { + "name": "nix:fod:method", + "value": "fetchzip" + }, + { + "name": "nix:fod:path", + "value": "/nix/store/pqwf2h9394hpp7195h977bv5zdnvf6z5-source" + }, + { + "name": "nix:fod:rev", + "value": "cdc59b76415730b46107a842b79c31d3ecbf48ea" + }, + { + "name": "nix:fod:sha256", + "value": "1cj31y9y3ibq5sa08v01i0fccmw1lr52v70x8brs44mr8ckc9bfp" + }, + { + "name": "nix:fod:url", + "value": "https://git.syndicate-lang.org/ehmry/syndicate-nim/archive/cdc59b76415730b46107a842b79c31d3ecbf48ea.tar.gz" + }, + { + "name": "nix:fod:ref", + "value": "20240623" + }, + { + "name": "nix:fod:srcDir", + "value": "src" + } + ] + }, + { + "type": "library", + "bom-ref": "pkg:nim/nimcrypto", + "name": "nimcrypto", + "version": "485f7b3cfa83c1beecc0e31be0e964d697aa74d7", + "externalReferences": [ + { + "url": "https://github.com/cheatfate/nimcrypto/archive/485f7b3cfa83c1beecc0e31be0e964d697aa74d7.tar.gz", + "type": "source-distribution" + }, + { + "url": "https://github.com/cheatfate/nimcrypto", + "type": "vcs" + } + ], + "properties": [ + { + "name": "nix:fod:method", + "value": "fetchzip" + }, + { + "name": "nix:fod:path", + "value": "/nix/store/fkrcpp8lzj2yi21na79xm63xk0ggnqsp-source" + }, + { + "name": "nix:fod:rev", + "value": "485f7b3cfa83c1beecc0e31be0e964d697aa74d7" + }, + { + "name": "nix:fod:sha256", + "value": "1h3dzdbc9kacwpi10mj73yjglvn7kbizj1x8qc9099ax091cj5xn" + }, + { + "name": "nix:fod:url", + "value": "https://github.com/cheatfate/nimcrypto/archive/485f7b3cfa83c1beecc0e31be0e964d697aa74d7.tar.gz" + } + ] + }, + { + "type": "library", + "bom-ref": "pkg:nim/preserves", + "name": "preserves", + "version": "20240623", + "externalReferences": [ + { + "url": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/19f5641ec0459d60f0ae992aa72151c45a73ef2f.tar.gz", + "type": "source-distribution" + }, + { + "url": "https://git.syndicate-lang.org/ehmry/preserves-nim.git", + "type": "vcs" + } + ], + "properties": [ + { + "name": "nix:fod:method", + "value": "fetchzip" + }, + { + "name": "nix:fod:path", + "value": "/nix/store/l2xg6y34x7h9sxbx8fk61j9m4ramqaac-source" + }, + { + "name": "nix:fod:rev", + "value": "19f5641ec0459d60f0ae992aa72151c45a73ef2f" + }, + { + "name": "nix:fod:sha256", + "value": "08q86d7s96wy8190z2973bb0cjp5k6xdakna1669672m1davqxnl" + }, + { + "name": "nix:fod:url", + "value": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/19f5641ec0459d60f0ae992aa72151c45a73ef2f.tar.gz" + }, + { + "name": "nix:fod:ref", + "value": "20240623" + }, + { + "name": "nix:fod:srcDir", + "value": "src" + } + ] + }, + { + "type": "library", + "bom-ref": "pkg:nim/sys", + "name": "sys", + "version": "4ef3b624db86e331ba334e705c1aa235d55b05e1", + "externalReferences": [ + { + "url": "https://github.com/ehmry/nim-sys/archive/4ef3b624db86e331ba334e705c1aa235d55b05e1.tar.gz", + "type": "source-distribution" + }, + { + "url": "https://github.com/ehmry/nim-sys.git", + "type": "vcs" + } + ], + "properties": [ + { + "name": "nix:fod:method", + "value": "fetchzip" + }, + { + "name": "nix:fod:path", + "value": "/nix/store/syhxsjlsdqfap0hk4qp3s6kayk8cqknd-source" + }, + { + "name": "nix:fod:rev", + "value": "4ef3b624db86e331ba334e705c1aa235d55b05e1" + }, + { + "name": "nix:fod:sha256", + "value": "1q4qgw4an4mmmcbx48l6xk1jig1vc8p9cq9dbx39kpnb0890j32q" + }, + { + "name": "nix:fod:url", + "value": "https://github.com/ehmry/nim-sys/archive/4ef3b624db86e331ba334e705c1aa235d55b05e1.tar.gz" + }, + { + "name": "nix:fod:srcDir", + "value": "src" + } + ] + }, + { + "type": "library", + "bom-ref": "pkg:nim/taps", + "name": "taps", + "version": "20240405", + "externalReferences": [ + { + "url": "https://git.sr.ht/~ehmry/nim_taps/archive/8c8572cd971d1283e6621006b310993c632da247.tar.gz", + "type": "source-distribution" + }, + { + "url": "https://git.sr.ht/~ehmry/nim_taps", + "type": "vcs" + } + ], + "properties": [ + { + "name": "nix:fod:method", + "value": "fetchzip" + }, + { + "name": "nix:fod:path", + "value": "/nix/store/6y14ia52kr7jyaa0izx37mlablmq9s65-source" + }, + { + "name": "nix:fod:rev", + "value": "8c8572cd971d1283e6621006b310993c632da247" + }, + { + "name": "nix:fod:sha256", + "value": "1dp166bv9x773jmfqppg5i3v3rilgff013vb11yzwcid9l7s3iy8" + }, + { + "name": "nix:fod:url", + "value": "https://git.sr.ht/~ehmry/nim_taps/archive/8c8572cd971d1283e6621006b310993c632da247.tar.gz" + }, + { + "name": "nix:fod:ref", + "value": "20240405" + }, + { + "name": "nix:fod:srcDir", + "value": "src" + } + ] + }, + { + "type": "library", + "bom-ref": "pkg:nim/coap", + "name": "coap", + "version": "20230331", + "externalReferences": [ + { + "url": "https://codeberg.org/eris/nim-coap/archive/a134213b51a8d250684f2ba26802ffa97fae4ffb.tar.gz", + "type": "source-distribution" + }, + { + "url": "https://codeberg.org/eris/nim-coap.git", + "type": "vcs" + } + ], + "properties": [ + { + "name": "nix:fod:method", + "value": "fetchzip" + }, + { + "name": "nix:fod:path", + "value": "/nix/store/pqj933cnw7r7hp46jrpjlwh1yr0jvckp-source" + }, + { + "name": "nix:fod:rev", + "value": "a134213b51a8d250684f2ba26802ffa97fae4ffb" + }, + { + "name": "nix:fod:sha256", + "value": "1wbix6d8l26nj7m3xinh4m2f27n4ma0yzs3x5lpann2ha0y51k8b" + }, + { + "name": "nix:fod:url", + "value": "https://codeberg.org/eris/nim-coap/archive/a134213b51a8d250684f2ba26802ffa97fae4ffb.tar.gz" + }, + { + "name": "nix:fod:ref", + "value": "20230331" + }, + { + "name": "nix:fod:srcDir", + "value": "src" + } + ] + }, + { + "type": "library", + "bom-ref": "pkg:nim/npeg", + "name": "npeg", + "version": "1.2.2", + "externalReferences": [ + { + "url": "https://github.com/zevv/npeg/archive/ec0cc6e64ea4c62d2aa382b176a4838474238f8d.tar.gz", + "type": "source-distribution" + }, + { + "url": "https://github.com/zevv/npeg.git", + "type": "vcs" + } + ], + "properties": [ + { + "name": "nix:fod:method", + "value": "fetchzip" + }, + { + "name": "nix:fod:path", + "value": "/nix/store/xpn694ibgipj8xak3j4bky6b3k0vp7hh-source" + }, + { + "name": "nix:fod:rev", + "value": "ec0cc6e64ea4c62d2aa382b176a4838474238f8d" + }, + { + "name": "nix:fod:sha256", + "value": "1fi9ls3xl20bmv1ikillxywl96i9al6zmmxrbffx448gbrxs86kg" + }, + { + "name": "nix:fod:url", + "value": "https://github.com/zevv/npeg/archive/ec0cc6e64ea4c62d2aa382b176a4838474238f8d.tar.gz" + }, + { + "name": "nix:fod:ref", + "value": "1.2.2" + }, + { + "name": "nix:fod:srcDir", + "value": "src" + } + ] + }, + { + "type": "library", + "bom-ref": "pkg:nim/bigints", + "name": "bigints", + "version": "20231006", + "externalReferences": [ + { + "url": "https://github.com/ehmry/nim-bigints/archive/86ea14d31eea9275e1408ca34e6bfe9c99989a96.tar.gz", + "type": "source-distribution" + }, + { + "url": "https://github.com/ehmry/nim-bigints.git", + "type": "vcs" + } + ], + "properties": [ + { + "name": "nix:fod:method", + "value": "fetchzip" + }, + { + "name": "nix:fod:path", + "value": "/nix/store/jvrm392g8adfsgf36prgwkbyd7vh5jsw-source" + }, + { + "name": "nix:fod:rev", + "value": "86ea14d31eea9275e1408ca34e6bfe9c99989a96" + }, + { + "name": "nix:fod:sha256", + "value": "15pcpmnk1bnw3k8769rjzcpg00nahyrypwbxs88jnwr4aczp99j4" + }, + { + "name": "nix:fod:url", + "value": "https://github.com/ehmry/nim-bigints/archive/86ea14d31eea9275e1408ca34e6bfe9c99989a96.tar.gz" + }, + { + "name": "nix:fod:ref", + "value": "20231006" + }, + { + "name": "nix:fod:srcDir", + "value": "src" + } + ] + }, + { + "type": "library", + "bom-ref": "pkg:nim/cps", + "name": "cps", + "version": "0.10.4", + "externalReferences": [ + { + "url": "https://github.com/nim-works/cps/archive/2a4d771a715ba45cfba3a82fa625ae7ad6591c8b.tar.gz", + "type": "source-distribution" + }, + { + "url": "https://github.com/nim-works/cps", + "type": "vcs" + } + ], + "properties": [ + { + "name": "nix:fod:method", + "value": "fetchzip" + }, + { + "name": "nix:fod:path", + "value": "/nix/store/m9vpcf3dq6z2h1xpi1vlw0ycxp91s5p7-source" + }, + { + "name": "nix:fod:rev", + "value": "2a4d771a715ba45cfba3a82fa625ae7ad6591c8b" + }, + { + "name": "nix:fod:sha256", + "value": "0c62k5wpq9z9mn8cd4rm8jjc4z0xmnak4piyj5dsfbyj6sbdw2bf" + }, + { + "name": "nix:fod:url", + "value": "https://github.com/nim-works/cps/archive/2a4d771a715ba45cfba3a82fa625ae7ad6591c8b.tar.gz" + }, + { + "name": "nix:fod:ref", + "value": "0.10.4" + } + ] + }, + { + "type": "library", + "bom-ref": "pkg:nim/stew", + "name": "stew", + "version": "3c91b8694e15137a81ec7db37c6c58194ec94a6a", + "externalReferences": [ + { + "url": "https://github.com/status-im/nim-stew/archive/3c91b8694e15137a81ec7db37c6c58194ec94a6a.tar.gz", + "type": "source-distribution" + }, + { + "url": "https://github.com/status-im/nim-stew", + "type": "vcs" + } + ], + "properties": [ + { + "name": "nix:fod:method", + "value": "fetchzip" + }, + { + "name": "nix:fod:path", + "value": "/nix/store/mqg8qzsbcc8xqabq2yzvlhvcyqypk72c-source" + }, + { + "name": "nix:fod:rev", + "value": "3c91b8694e15137a81ec7db37c6c58194ec94a6a" + }, + { + "name": "nix:fod:sha256", + "value": "17lfhfxp5nxvld78xa83p258y80ks5jb4n53152cdr57xk86y07w" + }, + { + "name": "nix:fod:url", + "value": "https://github.com/status-im/nim-stew/archive/3c91b8694e15137a81ec7db37c6c58194ec94a6a.tar.gz" + } + ] + }, + { + "type": "library", + "bom-ref": "pkg:nim/getdns", + "name": "getdns", + "version": "20230806", + "externalReferences": [ + { + "url": "https://git.sr.ht/~ehmry/getdns-nim/archive/e4ae0992ed7c5540e6d498f3074d06c8f454a0b6.tar.gz", + "type": "source-distribution" + }, + { + "url": "https://git.sr.ht/~ehmry/getdns-nim", + "type": "vcs" + } + ], + "properties": [ + { + "name": "nix:fod:method", + "value": "fetchzip" + }, + { + "name": "nix:fod:path", + "value": "/nix/store/j8i20k9aarzppg4p234449140nnnaycq-source" + }, + { + "name": "nix:fod:rev", + "value": "e4ae0992ed7c5540e6d498f3074d06c8f454a0b6" + }, + { + "name": "nix:fod:sha256", + "value": "1dp53gndr6d9s9601dd5ipkiq94j53hlx46mxv8gpr8nd98bqysg" + }, + { + "name": "nix:fod:url", + "value": "https://git.sr.ht/~ehmry/getdns-nim/archive/e4ae0992ed7c5540e6d498f3074d06c8f454a0b6.tar.gz" + }, + { + "name": "nix:fod:ref", + "value": "20230806" + }, + { + "name": "nix:fod:srcDir", + "value": "src" + } + ] + } + ], + "dependencies": [ + { + "ref": "pkg:nim/nix_actor", + "dependsOn": [ + "pkg:nim/syndicate" + ] + }, + { + "ref": "pkg:nim/syndicate", + "dependsOn": [ + "pkg:nim/nimcrypto", + "pkg:nim/preserves", + "pkg:nim/sys", + "pkg:nim/taps" + ] + }, + { + "ref": "pkg:nim/nimcrypto", + "dependsOn": [] + }, + { + "ref": "pkg:nim/preserves", + "dependsOn": [ + "pkg:nim/npeg", + "pkg:nim/bigints" + ] + }, + { + "ref": "pkg:nim/sys", + "dependsOn": [ + "pkg:nim/cps", + "pkg:nim/stew" + ] + }, + { + "ref": "pkg:nim/taps", + "dependsOn": [ + "pkg:nim/getdns", + "pkg:nim/sys", + "pkg:nim/cps" + ] + }, + { + "ref": "pkg:nim/coap", + "dependsOn": [ + "pkg:nim/taps" + ] + }, + { + "ref": "pkg:nim/npeg", + "dependsOn": [] + }, + { + "ref": "pkg:nim/bigints", + "dependsOn": [] + }, + { + "ref": "pkg:nim/cps", + "dependsOn": [] + }, + { + "ref": "pkg:nim/stew", + "dependsOn": [] + }, + { + "ref": "pkg:nim/getdns", + "dependsOn": [] + } + ] +} diff --git a/shell.nix b/shell.nix deleted file mode 100644 index 75d225f97474..000000000000 --- a/shell.nix +++ /dev/null @@ -1,17 +0,0 @@ -{ pkgs ? import { } }: - -let - nix' = pkgs.nix.overrideAttrs (final: prev: { - src = pkgs.fetchFromGitHub { - owner = "tweag"; - repo = "nix"; - rev = "nix-c-bindings"; - hash = "sha256-xOyU79lsz0THOj1LccfsDS45089n2DhlkWxaJFeKriY="; - }; - }); -in pkgs.buildNimPackage { - name = "dummy"; - nativeBuildInputs = [ pkgs.pkg-config ]; - buildInputs = [ pkgs.boost nix' ]; - lockFile = ./lock.json; -} diff --git a/src/Tupfile b/src/Tupfile deleted file mode 100644 index 7bd661c4a4c8..000000000000 --- a/src/Tupfile +++ /dev/null @@ -1,3 +0,0 @@ -include_rules -: foreach *.nim | $(SYNDICATE_PROTOCOL) ./ |> !nim_bin |> {bin} -: foreach {bin} |> !assert_built |> diff --git a/src/nix_actor.nim b/src/nix_actor.nim index b4e58e909674..dc9f392f39bc 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -1,81 +1,200 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -import std/[os, strutils, tables] -import preserves, syndicate, syndicate/relays -# from syndicate/protocols/dataspace import Observe -import ./nix_actor/[nix_api, nix_api_value] -import ./nix_actor/protocol +import + std/[options, os, osproc, streams, strtabs, strutils, tables, times], + pkg/preserves, + pkg/syndicate, + pkg/syndicate/protocols/gatekeeper, + pkg/syndicate/relays, + ./nix_actor/[nix_api, nix_api_util, nix_api_value], + ./nix_actor/protocol -proc toPreserve(state: State; value: Value; E = void): Preserve[E] {.gcsafe.} = +proc echo(args: varargs[string, `$`]) {.used.} = + stderr.writeLine(args) + +type + Value = preserves.Value + NixValue = nix_api.Value + StringThunkRef = ref StringThunkObj + StringThunkObj = object of EmbeddedObj + data: Option[string] + +proc thunkString(start: cstring; n: cuint; state: pointer) {.cdecl.} = + let thunk = cast[ptr StringThunkObj](state) + assert thunk.data.isNone + var buf = newString(n) + copyMem(buf[0].addr, start, buf.len) + thunk.data = buf.move.some + +proc unthunk(v: Value): Value = + let thunk = v.unembed(StringThunkRef) + assert thunk.isSome + assert thunk.get.data.isSome + thunk.get.data.get.toPreserves + +proc toPreserves(value: NixValue; state: EvalState): Value {.gcsafe.} = var ctx: NixContext - stderr.writeLine get_type(ctx, value).int case get_type(ctx, value) of NIX_TYPE_THUNK: raiseAssert "cannot preserve thunk" of NIX_TYPE_INT: - result = getInt(ctx, value).toPreserve(E) + result = getInt(ctx, value).toPreserves of NIX_TYPE_FLOAT: - result = getFloat(ctx, value).toPreserve(E) + result = getFloat(ctx, value).toPreserves of NIX_TYPE_BOOL: - result = getBool(ctx, value).toPreserve(E) + result = getBool(ctx, value).toPreserves of NIX_TYPE_STRING: - result = ($getString(ctx, value)).toPreserve(E) + let thunk = StringThunkRef() + let err = getString(ctx, value, thunkString, thunk[].addr) + doAssert err == NIX_OK, $err + result = thunk.embed of NIX_TYPE_PATH: - result = ($getPathString(ctx, value)).toPreserve(E) + result = ($getPathString(ctx, value)).toPreserves of NIX_TYPE_NULL: - result = initRecord[E]("null") + result = initRecord("null") of NIX_TYPE_ATTRS: - result = initDictionary(E) + result = initDictionary() let n = getAttrsSize(ctx, value) var i: cuint while i < n: var (key, val) = get_attr_byidx(ctx, value, state, i) inc(i) - result[toSymbol($key, E)] = toPreserve(state, val, E) - stderr.writeLine(result) + result[toSymbol($key)] = val.toPreserves(state) # close(val) of NIX_TYPE_LIST: let n = getListSize(ctx, value) - result = initSequence(n, E) + result = initSequence(n) var i: cuint while i < n: var val = getListByIdx(ctx, value, state, i) - result[i] = toPreserve(state, val, E) + result[i] = val.toPreserves(state) inc(i) # close(val) of NIX_TYPE_FUNCTION, NIX_TYPE_EXTERNAL: raiseAssert "TODO: need a failure type" -type - BootArgs {.preservesDictionary.} = object - dataspace: Cap +proc findCommand(detail: ResolveDetail; cmd: string): string = + for dir in detail.`command-path`: + result = dir / cmd + if result.fileExists: + return + raise newException(OSError, "could not find " & cmd) + +proc commandlineArgs(detail: ResolveDetail; args: varargs[string]): seq[string] = + result = newSeqOfCap[string](detail.options.len * 2 + args.len) + for sym, val in detail.options: + result.add("--" & $sym) + if not val.isString "": + result.add(val.jsonText) + for arg in args: + result.add arg + +proc commandlineEnv(detail: ResolveDetail): StringTableRef = + newStringTable({"NIX_PATH": detail.lookupPath.join ":"}) + +proc instantiate(facet: Facet; detail: ResolveDetail; expr: string; log: Option[Cap]): InstantiateResult = + # TODO: run this process asynchronously with nim-sys. + var p: Process + try: + p = startProcess( + detail.findCommand("nix-instantiate"), + args = detail.commandlineArgs("--expr", expr), + env = detail.commandlineEnv(), + options = {}, + ) + var + errors = errorStream(p) + line = "".toPreserves + while true: + if errors.readLine(line.string): + if log.isSome: + facet.run do (turn: Turn): + message(turn, log.get, line) + elif not running(p): break + initDuration(milliseconds = 250).some.runOnce + var path = p.outputStream.readAll.strip + if path != "": + result = InstantiateResult(orKind: InstantiateResultKind.Derivation) + result.derivation.expr = expr + result.derivation.storePath = path + except CatchableError as err: + reset result + result.error.message = err.msg + finally: + close(p) + +proc realise(facet: Facet; detail: ResolveDetail; drv: string; log: Option[Cap]): RealiseResult = + # TODO: run this process asynchronously with nim-sys. + var p: Process + try: + p = startProcess( + detail.findCommand("nix-store"), + args = detail.commandlineArgs("--realise", drv), + env = detail.commandlineEnv(), + options = {}, + ) + var + errors = errorStream(p) + line = "".toPreserves + while true: + if errors.readLine(line.string): + if log.isSome: + facet.run do (turn: Turn): + message(turn, log.get, line) + elif not running(p): break + initDuration(milliseconds = 250).some.runOnce + var storePaths = p.outputStream.readAll.strip.split + if storePaths != @[]: + result = RealiseResult(orKind: RealiseResultKind.Outputs) + result.outputs.drv = drv + result.outputs.storePaths = storePaths + except CatchableError as err: + reset result + result.error.message = err.msg + finally: + close(p) + +proc eval(store: Store; state: EvalState; expr: string): EvalResult = + var nixVal: NixValue + try: + nixVal = state.evalFromString(expr, "") + state.force(nixVal) + result = EvalResult(orKind: EvalResultKind.EvalSuccess) + result.evalsuccess.expr = expr + result.evalsuccess.result = nixVal.toPreserves(state).mapEmbeds(unthunk) + except CatchableError as err: + reset result + result.error.message = err.msg + finally: + close(nixVal) + +proc serve(turn: Turn; detail: ResolveDetail; store: Store; state: EvalState; ds: Cap) = + during(turn, ds, Eval.grabWithin) do (expr: string, log: Cap, resp: Cap): + discard publish(turn, resp, eval(store, state, expr)) + + during(turn, ds, Instantiate.grabWithin) do (expr: string, log: Value, resp: Cap): + discard publish(turn, resp, instantiate(turn.facet, detail, expr, log.unembed(Cap))) + + during(turn, ds, Realise.grabWithin) do (drv: string, log: Value, resp: Cap): + discard publish(turn, resp, realise(turn.facet, detail, drv, log.unembed(Cap))) proc main() = initLibexpr() - runActor("nix_actor") do (root: Cap; turn: var Turn): - connectStdio(turn, root) - - during(turn, root, ?BootArgs) do (ds: Cap): - let - store = openStore() - state = newState(store) - - let pat = ?Observe(pattern: !Eval) ?? {0: grabLit(), 1: grabLit()} - during(turn, ds, pat) do (expr: string, path: string): - var - value: Value - ass = Eval(expr: expr, path: path) - try: - value = evalFromString(state, ass.expr, ass.path) - force(state, value) - ass.result = toPreserve(state, value, void) - discard publish(turn, ds, ass) - except CatchableError as err: - stderr.writeLine "failed to evaluate ", ass.expr, ": ", err.msg - close(value) - do: - close(state) - close(store) + runActor("main") do (turn: Turn): + resolveEnvironment(turn) do (turn: Turn; relay: Cap): + let pat = Resolve?:{ 0: ResolveStep.grabWithin, 1: grab() } + during(turn, relay, pat) do (detail: ResolveDetail; observer: Cap): + let + store = openStore() + state = newState(store, detail.lookupPath) + ds = turn.newDataspace() + # TODO: attenuate this dataspace to only the assertions we observe. + linkActor(turn, "nix-actor") do (turn: Turn): + serve(turn, detail, store, state, ds) + discard publish(turn, observer, ResolvedAccepted(responderSession: ds)) + do: + close(state) + close(store) main() diff --git a/src/nix_actor/Tupfile b/src/nix_actor/Tupfile index 530ef5039c59..81799a46b0a8 100644 --- a/src/nix_actor/Tupfile +++ b/src/nix_actor/Tupfile @@ -1,2 +1,2 @@ include_rules -: ../../protocol.prs |> !preserves_schema_nim |> protocol.nim | ../ +: $(PROJECT_DIR)/protocol.prs |> !preserves-schema-nim |> protocol.nim | $(PROJECT_DIR)/ diff --git a/src/nix_actor/clients.nim b/src/nix_actor/clients.nim deleted file mode 100644 index b227ac332559..000000000000 --- a/src/nix_actor/clients.nim +++ /dev/null @@ -1,174 +0,0 @@ -# SPDX-FileCopyrightText: ☭ Emery Hemingway -# SPDX-License-Identifier: Unlicense - -import std/[asyncdispatch, asyncnet, os, sets, strutils, tables] -from std/algorithm import sort - -import eris -import preserves, syndicate -import ./protocol, ./sockets - -proc sendNext(client: Session; msg: string) {.async.} = - await send(client, STDERR_NEXT) - await send(client, msg) - -proc sendWorkEnd(client: Session): Future[void] = - send(client, STDERR_LAST) - -proc send(client: Session; miss: Missing) {.async.} = - await sendWorkEnd(client) - await send(client, miss.willBuild) - await send(client, miss.willSubstitute) - await send(client, miss.unknown) - await send(client, Word miss.downloadSize) - await send(client, Word miss.narSize) - -proc send(client: Session; info: LegacyPathAttrs) {.async.} = - await send(client, info.deriver) - await send(client, info.narHash) - await send(client, info.references) - await send(client, Word info.registrationTime) - await send(client, Word info.narSize) - await send(client, Word info.ultimate) - await send(client, info.sigs) - await send(client, info.ca) - -proc sendValidInfo(client: Session; info: LegacyPathAttrs) {.async.} = - await sendWorkEnd(client) - await send(client, 1) # valid - await send(client, info) - -proc completeAddToStore(client: Session; path: string; info: LegacyPathAttrs) {.async.} = - await sendWorkEnd(client) - await send(client, path) - await send(client, info) - -proc serveClient(facet: Facet; ds: Cap; store: ErisStore; client: Session) {.async.} = - block: - let clientMagic = await recvWord(client) - if clientMagic != WORKER_MAGIC_1: - raise newException(ProtocolError, "invalid protocol magic") - await send(client, WORKER_MAGIC_2, PROTOCOL_VERSION) - let clientVersion = Version(await recvWord(client)) - if clientVersion < 0x1_21: - raise newException(ProtocolError, "obsolete protocol version") - assert clientVersion.minor >= 14 - discard await(recvWord(client)) - # obsolete CPU affinity - assert clientVersion.minor >= 11 - discard await(recvWord(client)) - # obsolete reserveSpace - assert clientVersion.minor >= 33 - await send(client, "0.0.0") - await sendWorkEnd(client) - while not client.socket.isClosed: - let - w = await recvWord(client.socket) - wop = WorkerOperation(w) - case wop - - of wopAddToStore: - let - name = await recvString(client) - caMethod = await recvString(client) - var storeRefs = await recvStringSeq(client) - sort(storeRefs) # sets not valid for patterns so use a sorted list - discard await recvWord(client) # repair, not implemented - let cap = await ingestChunks(client, store) - await sendNext(client, $cap & " " & name) - let attrsPat = inject(?AddToStoreAttrs, { - "name".toSymbol(Cap): ?name, - "ca-method".toSymbol(Cap): ?caMethod.toSymbol, - "references".toSymbol(Cap): ?storeRefs, - "eris".toSymbol(Cap): ?cap.bytes, - }) - # bind AddToStoreAttrs and override with some literal values - let pat = PathInfo ? { 0: grab(), 1: attrsPat } - run(facet) do (turn: var Turn): - onPublish(turn, ds, pat) do (path: string, ca: string, deriver: string, narHash: string, narSize: BiggestInt, regTime: BiggestInt, sigs: StringSet, ultimate: bool): - asyncCheck(turn, completeAddToStore(client, path, LegacyPathAttrs( - ca: ca, - deriver: deriver, - narHash: narHash, - narSize: narSize, - references: storeRefs, - registrationTime: regTime, - sigs: sigs, - ultimate: ultimate, - ))) - - of wopQueryPathInfo: - let - path = await recvString(client) - pat = PathInfo ? { 0: ?path, 1: grab() } - run(facet) do (turn: var Turn): - onPublish(turn, ds, pat) do (info: LegacyPathAttrs): - asyncCheck(turn, sendValidInfo(client, info)) - - of wopQueryMissing: - var targets = toPreserve(await recvStringSeq(client)) - sort(targets.sequence) - # would prefer to use a set but that doesn't translate into a pattern - let pat = inject(?Missing, { 0: ?targets }) - run(facet) do (turn: var Turn): - onPublish(turn, ds, pat) do ( - willBuild: StringSet, - willSubstitute: StringSet, - unknown: StringSet, - downloadSize: BiggestInt, - narSize: BiggestInt - ): - let miss = Missing( - willBuild: willBuild, - willSubstitute: willSubstitute, - unknown: unknown, - downloadSize: downloadSize, - narSize: narSize, - ) - asyncCheck(turn, send(client, miss)) - - of wopSetOptions: - await discardWords(client, 12) - # 01 keepFailed - # 02 keepGoing - # 03 tryFallback - # 04 verbosity - # 05 maxBuildJobs - # 06 maxSilentTime - # 07 useBuildHook - # 08 verboseBuild - # 09 logType - # 10 printBuildTrace - # 11 buildCores - # 12 useSubstitutes - let overridePairCount = await recvWord(client) - for _ in 1..overridePairCount: - discard await (recvString(client)) - discard await (recvString(client)) - await sendWorkEnd(client) - # all options from the client are ingored - - else: - let msg = "unhandled worker op " & $wop - await sendNext(client, msg) - await sendWorkEnd(client) - close(client.socket) - -proc serveClientSide(facet: Facet; ds: Cap; store: ErisStore; listener: AsyncSocket) {.async.} = - while not listener.isClosed: - let - client = await accept(listener) - fut = serveClient(facet, ds, store, newSession(client)) - addCallback(fut) do (): - if not client.isClosed: - close(client) - -proc bootClientSide*(turn: var Turn; ds: Cap; store: ErisStore; socketPath: string) = - let listener = newUnixSocket() - onStop(turn.facet) do (turn: var Turn): - close(listener) - removeFile(socketPath) - removeFile(socketPath) - bindUnix(listener, socketPath) - listen(listener) - asyncCheck(turn, serveClientSide(turn.facet, ds, store, listener)) diff --git a/src/nix_actor/daemons.nim b/src/nix_actor/daemons.nim deleted file mode 100644 index 8e3a7fbb0a4f..000000000000 --- a/src/nix_actor/daemons.nim +++ /dev/null @@ -1,188 +0,0 @@ -# SPDX-FileCopyrightText: ☭ Emery Hemingway -# SPDX-License-Identifier: Unlicense - -import std/[asyncdispatch, asyncnet, sets, strutils] -from std/algorithm import sort - -import eris -import preserves, syndicate -from syndicate/protocols/dataspace import Observe -import ./protocol, ./sockets - -type - Value = Preserve[void] - Observe = dataspace.Observe[Cap] - -proc recvError(daemon: Session): Future[string] {.async.} = - discard #[typ]# await recvString(daemon) - discard #[lvl]# await recvWord(daemon) - discard #[name]# await recvString(daemon) - let msg = #[msg]# await recvString(daemon) - discard #[havePos]# await recvWord(daemon) - let nrTraces = await recvWord(daemon) - for i in 1..nrTraces: - discard #[havPos]# await recvWord(daemon) - discard #[msg]# await recvString(daemon) - return msg - -proc recvFields(daemon: Session) {.async.} = - let count = await recvWord(daemon) - for i in 0..= 33: - discard await recvString(daemon) # version - if daemon.version.minor >= 35: - discard await recvWord(daemon) # remoteTrustsUs - await recvWork(daemon) - -proc queryMissing(daemon: Session; targets: StringSeq): Future[Missing] {.async.} = - var miss = Missing(targets: targets) - await send(daemon, Word wopQueryMissing) - await send(daemon, miss.targets) - await recvWork(daemon) - miss.willBuild = await recvStringSet(daemon) - miss.willSubstitute = await recvStringSet(daemon) - miss.unknown = await recvStringSet(daemon) - miss.downloadSize = BiggestInt await recvWord(daemon) - miss.narSize = BiggestInt await recvWord(daemon) - return miss - -proc queryPathInfo(daemon: Session; path: string): Future[LegacyPathAttrs] {.async.} = - var info: LegacyPathAttrs - await send(daemon, Word wopQueryPathInfo) - await send(daemon, path) - await recvWork(daemon) - let valid = await recvWord(daemon) - if valid != 0: - info.deriver = await recvString(daemon) - info.narHash = await recvString(daemon) - info.references = await recvStringSeq(daemon) - sort(info.references) - info.registrationTime = BiggestInt await recvWord(daemon) - info.narSize = BiggestInt await recvWord(daemon) - info.ultimate = (await recvWord(daemon)) != 0 - info.sigs = await recvStringSet(daemon) - info.ca = await recvString(daemon) - return info - -proc recvLegacyPathAttrs(daemon: Session): Future[AddToStoreAttrs] {.async.} = - var info: AddToStoreAttrs - info.deriver = await recvString(daemon) - info.narHash = await recvString(daemon) - info.references = await recvStringSeq(daemon) - sort(info.references) - info.registrationTime = BiggestInt await recvWord(daemon) - info.narSize = BiggestInt await recvWord(daemon) - assert daemon.version.minor >= 16 - info.ultimate = (await recvWord(daemon)) != 0 - info.sigs = await recvStringSet(daemon) - info.ca = await recvString(daemon) - return info - -proc addToStore(daemon: Session; store: ErisStore; request: AddToStoreClientAttrs): Future[(string, AddToStoreAttrs)] {.async.} = - let erisCap = parseCap(request.eris) - await send(daemon, Word wopAddToStore) - await send(daemon, request.name) - await send(daemon, string request.`ca-method`) - await send(daemon, request.references) - await send(daemon, 0) # repair - await recoverChunks(daemon, store, erisCap) - await recvWork(daemon) - let path = await recvString(daemon) - var info = await recvLegacyPathAttrs(daemon) - info.eris = request.eris - info.`ca-method` = request.`ca-method` - info.name = request.name - info.references = request.references - return (path, info) - -proc callDaemon(turn: var Turn; path: string; action: proc (daemon: Session; turn: var Turn) {.gcsafe.}): Session = - let - daemon = newSession() - fut = connectDaemon(daemon, path) - addCallback(fut, turn) do (turn: var Turn): - read(fut) - action(daemon, turn) - return daemon - -proc bootDaemonSide*(turn: var Turn; ds: Cap; store: ErisStore; socketPath: string) = - - during(turn, ds, ?Observe(pattern: !Missing) ?? {0: grab()}) do (targets: Literal[StringSeq]): - # cannot use `grabLit` here because an array is a compound - # TODO: unpack to a `Pattern` - let daemon = callDaemon(turn, socketPath) do (daemon: Session; turn: var Turn): - let missFut = queryMissing(daemon, targets.value) - addCallback(missFut, turn) do (turn: var Turn): - close(daemon) - var miss = read(missFut) - discard publish(turn, ds, miss) - do: - close(daemon) - - during(turn, ds, ?Observe(pattern: !PathInfo) ?? {0: grabLit()}) do (path: string): - let daemon = callDaemon(turn, socketPath) do (daemon: Session; turn: var Turn): - let infoFut = queryPathInfo(daemon, path) - addCallback(infoFut, turn) do (turn: var Turn): - close(daemon) - var info = read(infoFut) - discard publish(turn, ds, initRecord("path", path.toPreserve, info.toPreserve)) - do: - close(daemon) - - during(turn, ds, ?Observe(pattern: !PathInfo) ?? {1: grabDict()}) do (request: Literal[AddToStoreClientAttrs]): - let daemon = callDaemon(turn, socketPath) do (daemon: Session; turn: var Turn): - let fut = addToStore(daemon, store, request.value) - addCallback(fut, turn) do (turn: var Turn): - close(daemon) - var (path, info) = read(fut) - discard publish(turn, ds, initRecord("path", path.toPreserve, info.toPreserve)) - do: - close(daemon) diff --git a/src/nix_actor/nix_api.nim b/src/nix_actor/nix_api.nim index 2093773899ce..5486b32cb1ed 100644 --- a/src/nix_actor/nix_api.nim +++ b/src/nix_actor/nix_api.nim @@ -1,9 +1,9 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -import ./nix_api_types, ./nix_api_util, ./nix_api_value, ./nix_api_store, ./nix_api_expr +import ./nix_api_types, ./nix_api_value, ./nix_api_store, ./nix_api_expr -export NixContext, Store, State, Value, ValueType, +export NixContext, Store, EvalState, Value, ValueType, gc_decref, isNil {.passC: staticExec("pkg-config --cflags nix-expr-c").} @@ -25,21 +25,21 @@ proc openStore*(): Store = var ctx: NixContext result = store_open(ctx, nil, nil) -proc close*(store: Store) = store_unref(store) +proc close*(store: Store) = store_free(store) -proc newState*(store: Store; searchPath: varargs[string]): State = +proc newState*(store: Store; lookupPath: openarray[string]): EvalState = var ctx: NixContext - var path = allocCStringArray(searchPath) + var path = allocCStringArray(lookupPath) defer: deallocCStringArray(path) result = state_create(ctx, path, store) -proc close*(state: State) = state_free(state) +proc close*(state: EvalState) = state_free(state) -proc newValue*(state: State): Value = +proc newValue*(state: EvalState): Value = var ctx: NixContext alloc_value(ctx, state) -proc evalFromString*(state: State; expr, path: string): Value = +proc evalFromString*(state: EvalState; expr, path: string): Value = var ctx: NixContext result = alloc_value(ctx, state) discard expr_eval_from_string(ctx, state, expr, path, result) @@ -48,11 +48,10 @@ proc close*(value: Value) = var ctx: NixContext discard gc_decref(ctx, cast[pointer](value)) -proc force*(state: State; value: Value) = +proc force*(state: EvalState; value: Value) = var ctx: NixContext discard value_force(ctx, state, value) -proc get_attr_byidx*(ctx: NixContext; value: Value; state: State; i: cuint): (cstring, Value) = +proc get_attr_byidx*(ctx: NixContext; value: Value; state: EvalState; i: cuint): (cstring, Value) = var ctx: NixContext result[1] = get_attr_byidx(ctx, value, state, i, addr result[0]) - diff --git a/src/nix_actor/nix_api_expr.nim b/src/nix_actor/nix_api_expr.nim index d3cd6cc983e5..591a691ce8ed 100644 --- a/src/nix_actor/nix_api_expr.nim +++ b/src/nix_actor/nix_api_expr.nim @@ -1,4 +1,4 @@ -## Module generated by c2nim for nix_api_expr.h +## Module generated by c2nim from nix_api_expr.h import ./nix_api_types @@ -6,17 +6,19 @@ import ./nix_api_types proc libexpr_init*(context: NixContext): nix_err {.nix_api_expr.} -proc expr_eval_from_string*(context: NixContext; state: State; expr: cstring; path: cstring; value: Value): nix_err {.nix_api_expr.} +proc expr_eval_from_string*(context: NixContext; state: EvalState; expr: cstring; path: cstring; value: Value): nix_err {.nix_api_expr.} -proc value_call*(context: NixContext; state: State; fn: Value; arg: Value; value: Value): nix_err {.nix_api_expr.} +proc value_call*(context: NixContext; state: EvalState; fn: Value; arg: Value; value: Value): nix_err {.nix_api_expr.} -proc value_force*(context: NixContext; state: State; value: Value): nix_err {.nix_api_expr.} +proc value_call_multi*(context: NixContext; state: EvalState; fn: Value; nargs: csize_t; args: ptr UncheckedArray[Value]; value: Value): nix_err {.nix_api_expr.} -proc value_force_deep*(context: NixContext; state: State; value: Value): nix_err {.nix_api_expr.} +proc value_force*(context: NixContext; state: EvalState; value: Value): nix_err {.nix_api_expr.} -proc state_create*(context: NixContext; searchPath: cstringArray; store: Store): State {.nix_api_expr.} +proc value_force_deep*(context: NixContext; state: EvalState; value: Value): nix_err {.nix_api_expr.} -proc state_free*(state: State) {.nix_api_expr.} +proc state_create*(context: NixContext; lookupPath: cstringArray; store: Store): EvalState {.nix_api_expr.} + +proc state_free*(state: EvalState) {.nix_api_expr.} proc gc_incref*(context: NixContext; `object`: pointer): nix_err {.nix_api_expr.} @@ -25,4 +27,3 @@ proc gc_decref*(context: NixContext; `object`: pointer): nix_err {.nix_api_expr. proc gc_now*() {.nix_api_expr.} proc gc_register_finalizer*(obj: pointer; cd: pointer; finalizer: proc (obj: pointer; cd: pointer)) {.nix_api_expr.} - diff --git a/src/nix_actor/nix_api_store.nim b/src/nix_actor/nix_api_store.nim index 6e5ccfee0b71..904621041dfb 100644 --- a/src/nix_actor/nix_api_store.nim +++ b/src/nix_actor/nix_api_store.nim @@ -10,7 +10,7 @@ proc init_plugins*(context: NixContext): nix_err {.nix_api_store.} proc store_open*(a1: NixContext; uri: cstring; params: ptr cstringArray): Store {.nix_api_store.} -proc store_unref*(store: Store) {.nix_api_store.} +proc store_free*(store: Store) {.nix_api_store.} proc store_get_uri*(context: NixContext; store: Store; dest: cstring; n: cuint): nix_err {.nix_api_store.} diff --git a/src/nix_actor/nix_api_types.nim b/src/nix_actor/nix_api_types.nim new file mode 100644 index 000000000000..fcb25ba2ccfd --- /dev/null +++ b/src/nix_actor/nix_api_types.nim @@ -0,0 +1,25 @@ +type + nix_err* = cint + NixException* = object of CatchableError + NixContext* {.header: "nix_api_util.h", importc: "nix_c_context".} = distinct pointer + EvalState* {.header: "nix_api_expr.h", importc.} = distinct pointer + Store* {.header: "nix_api_store.h", importc.} = distinct pointer + StorePath* {.header: "nix_api_store.h", importc.} = distinct pointer + Value* {.header: "nix_api_value.h", importc.} = distinct pointer + ValueType* {.header: "nix_api_value.h", importc.} = enum + NIX_TYPE_THUNK, + NIX_TYPE_INT, + NIX_TYPE_FLOAT, + NIX_TYPE_BOOL, + NIX_TYPE_STRING, + NIX_TYPE_PATH, + NIX_TYPE_NULL, + NIX_TYPE_ATTRS, + NIX_TYPE_LIST, + NIX_TYPE_FUNCTION, + NIX_TYPE_EXTERNAL + +proc isNil*(p: NixContext): bool {.borrow.} +proc isNil*(p: EvalState): bool {.borrow.} +proc isNil*(p: Store): bool {.borrow.} +proc isNil*(p: Value): bool {.borrow.} diff --git a/src/nix_actor/nix_api_value.nim b/src/nix_actor/nix_api_value.nim index 057f6383a4fd..a93a981db3a2 100644 --- a/src/nix_actor/nix_api_value.nim +++ b/src/nix_actor/nix_api_value.nim @@ -2,71 +2,92 @@ import ./nix_api_types +{.pragma: nix_api_value, header: "nix_api_value.h", importc: "nix_$1".} + type - PrimOpFun* = proc (user_data: pointer; context: NixContext; state: ptr State; args: ptr Value; ret: Value) + BindingsBuilder* {.header: "nix_api_value.h", importc.} = distinct pointer + ExternalValue* {.header: "nix_api_value.h", importc.} = distinct pointer + ListBuilder* {.header: "nix_api_value.h", importc.} = distinct pointer + RealisedString* {.header: "nix_api_value.h", importc: "nix_realised_string".} = distinct pointer + GetStringCallback* = proc (start: cstring; n: cuint; data: pointer) {.cdecl.} + +proc alloc_value*(context: NixContext; state: EvalState): Value {.nix_api_value.} + +proc get_type*(context: NixContext; value: Value): ValueType {.nix_api_value.} + +proc get_typename*(context: NixContext; value: Value): cstring {.nix_api_value.} + +proc get_bool*(context: NixContext; value: Value): bool {.nix_api_value.} + +proc get_string*(context: NixContext; value: Value; callback: GetStringCallback; user_data: pointer): nix_err {.nix_api_value.} + +proc get_path_string*(context: NixContext; value: Value): cstring {.nix_api_value.} + +proc get_list_size*(context: NixContext; value: Value): cuint {.nix_api_value.} + +proc get_attrs_size*(context: NixContext; value: Value): cuint {.nix_api_value.} -# proc alloc_primop*(context: NixContext; fun: PrimOpFun; arity: cint; name: cstring; args: cstringArray; doc: cstring; user_data: pointer): ptr PrimOp {.importc: "nix_$1", header: "nix_api_value.h".} +proc get_float*(context: NixContext; value: Value): cdouble {.nix_api_value.} -# proc register_primop*(context: NixContext; primOp: ptr PrimOp): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} +proc get_int*(context: NixContext; value: Value): int64 {.nix_api_value.} -proc alloc_value*(context: NixContext; state: State): Value {.importc: "nix_$1", header: "nix_api_value.h".} +proc get_external*(context: NixContext; a2: Value): ExternalValue {.nix_api_value.} -proc get_type*(context: NixContext; value: Value): ValueType {.importc: "nix_$1", header: "nix_api_value.h".} +proc get_list_byidx*(context: NixContext; value: Value; state: EvalState; ix: cuint): Value {.nix_api_value.} -proc get_typename*(context: NixContext; value: Value): cstring {.importc: "nix_$1", header: "nix_api_value.h".} +proc get_attr_byname*(context: NixContext; value: Value; state: EvalState; name: cstring): Value {.nix_api_value.} -proc get_bool*(context: NixContext; value: Value): bool {.importc: "nix_$1", header: "nix_api_value.h".} +proc has_attr_byname*(context: NixContext; value: Value; state: EvalState; name: cstring): bool {.nix_api_value.} -proc get_string*(context: NixContext; value: Value): cstring {.importc: "nix_$1", header: "nix_api_value.h".} +proc get_attr_byidx*(context: NixContext; value: Value; state: EvalState; i: cuint; name: ptr cstring): Value {.nix_api_value.} -proc get_path_string*(context: NixContext; value: Value): cstring {.importc: "nix_$1", header: "nix_api_value.h".} +proc get_attr_name_byidx*(context: NixContext; value: Value; state: EvalState; i: cuint): cstring {.nix_api_value.} -proc get_list_size*(context: NixContext; value: Value): cuint {.importc: "nix_$1", header: "nix_api_value.h".} +proc init_bool*(context: NixContext; value: Value; b: bool): nix_err {.nix_api_value.} -proc get_attrs_size*(context: NixContext; value: Value): cuint {.importc: "nix_$1", header: "nix_api_value.h".} +proc init_string*(context: NixContext; value: Value; str: cstring): nix_err {.nix_api_value.} -proc get_float*(context: NixContext; value: Value): cdouble {.importc: "nix_$1", header: "nix_api_value.h".} +proc init_path_string*(context: NixContext; s: EvalState; value: Value; str: cstring): nix_err {.nix_api_value.} -proc get_int*(context: NixContext; value: Value): int64 {.importc: "nix_$1", header: "nix_api_value.h".} +proc init_float*(context: NixContext; value: Value; d: cdouble): nix_err {.nix_api_value.} -# proc get_external*(context: NixContext; a2: Value): ptr ExternalValue {.importc: "nix_$1", header: "nix_api_value.h".} +proc init_int*(context: NixContext; value: Value; i: int64): nix_err {.nix_api_value.} -proc get_list_byidx*(context: NixContext; value: Value; state: State; ix: cuint): Value {.importc: "nix_$1", header: "nix_api_value.h".} +proc init_null*(context: NixContext; value: Value): nix_err {.nix_api_value.} -proc get_attr_byname*(context: NixContext; value: Value; state: State; name: cstring): Value {.importc: "nix_$1", header: "nix_api_value.h".} +proc init_apply*(context: NixContext; value: Value; fn: Value; arg: Value): nix_err {.nix_api_value.} -proc has_attr_byname*(context: NixContext; value: Value; state: State; name: cstring): bool {.importc: "nix_$1", header: "nix_api_value.h".} +proc init_external*(context: NixContext; value: Value; val: ExternalValue): nix_err {.nix_api_value.} -proc get_attr_byidx*(context: NixContext; value: Value; state: State; i: cuint; name: ptr cstring): Value {.importc: "nix_$1", header: "nix_api_value.h".} +proc make_list*(context: NixContext; list_builder: ListBuilder; value: Value): nix_err {.nix_api_value.} -proc get_attr_name_byidx*(context: NixContext; value: Value; state: State; i: cuint): cstring {.importc: "nix_$1", header: "nix_api_value.h".} +proc make_list_builder*(context: NixContext; state: EvalState; capacity: csize_t): ListBuilder {.nix_api_value.} -proc set_bool*(context: NixContext; value: Value; b: bool): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} +proc list_builder_insert*(context: NixContext; list_builder: ListBuilder; index: cuint; value: Value): nix_err {.nix_api_value.} -proc set_string*(context: NixContext; value: Value; str: cstring): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} +proc list_builder_free*(list_builder: ListBuilder) {.nix_api_value.} -proc set_path_string*(context: NixContext; value: Value; str: cstring): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} +proc make_attrs*(context: NixContext; value: Value; b: BindingsBuilder): nix_err {.nix_api_value.} -proc set_float*(context: NixContext; value: Value; d: cdouble): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} +# proc init_primop*(context: NixContext; value: Value; op: PrimOp): nix_err {.nix_api_value.} -proc set_int*(context: NixContext; value: Value; i: int64): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} +proc copy_value*(context: NixContext; value: Value; source: Value): nix_err {.nix_api_value.} -proc set_null*(context: NixContext; value: Value): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} +proc make_bindings_builder*(context: NixContext; state: EvalState; capacity: csize_t): BindingsBuilder {.nix_api_value.} -# proc set_external*(context: NixContext; value: Value; val: ptr ExternalValue): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} +proc bindings_builder_insert*(context: NixContext; builder: BindingsBuilder; name: cstring; value: Value): nix_err {.nix_api_value.} -proc make_list*(context: NixContext; s: State; value: Value; size: cuint): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} +proc bindings_builder_free*(builder: BindingsBuilder) {.nix_api_value.} -proc set_list_byidx*(context: NixContext; value: Value; ix: cuint; elem: Value): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} +proc string_realise*(context: NixContext; state: EvalState; value: Value; isIFD: bool): RealisedString {.nix_api_value.} -# proc make_attrs*(context: NixContext; value: Value; b: ptr BindingsBuilder): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} +proc realised_string_get_buffer_start*(realised_string: RealisedString): cstring {.nix_api_value.} -# proc set_primop*(context: NixContext; value: Value; op: ptr PrimOp): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} +proc realised_string_get_buffer_size*(realised_string: RealisedString): csize_t {.nix_api_value.} -proc copy_value*(context: NixContext; value: Value; source: Value): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} +proc realised_string_get_store_path_count*(realised_string: RealisedString): csize_t {.nix_api_value.} -# proc make_bindings_builder*(context: NixContext; state: State; capacity: csize_t): ptr BindingsBuilder {.importc: "nix_$1", header: "nix_api_value.h".} +proc realised_string_get_store_path*(realised_string: RealisedString; index: csize_t): StorePath {.nix_api_value.} -# proc bindings_builder_insert*(context: NixContext; builder: ptr BindingsBuilder; name: cstring; value: Value): nix_err {.importc: "nix_$1", header: "nix_api_value.h".} +proc realised_string_free*(realised_string: RealisedString) {.nix_api_value.} -# proc bindings_builder_free*(builder: ptr BindingsBuilder) {.importc: "nix_$1", header: "nix_api_value.h".} diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 4a485f7b42b2..caa1330390ab 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -1,128 +1,96 @@ import - preserves, std/sets, std/tables + preserves, std/tables type + Error* {.preservesRecord: "error".} = object + `message`*: string + Eval* {.preservesRecord: "eval".} = object `expr`*: string - `path`*: string - `result`*: Preserve[void] + `log`* {.preservesEmbedded.}: Value + `result`*: EvalResult - AttrSet* = Table[Symbol, Preserve[void]] + AttrSet* = Table[Symbol, Value] Realise* {.preservesRecord: "realise".} = object `drv`*: string - `outputs`*: StringSeq - - LegacyPathAttrs* {.preservesDictionary.} = object - `ca`*: string - `deriver`*: string - `narHash`*: string - `narSize`*: BiggestInt - `references`*: StringSeq - `registrationTime`*: BiggestInt - `sigs`*: StringSet - `ultimate`*: bool - - Missing* {.preservesRecord: "missing".} = object - `targets`*: StringSeq - `willBuild`*: StringSet - `willSubstitute`*: StringSet - `unknown`*: StringSet - `downloadSize`*: BiggestInt - `narSize`*: BiggestInt - - Narinfo* {.preservesRecord: "narinfo".} = object - `path`*: string - `info`*: AttrSet - - FieldKind* {.pure.} = enum - `int`, `string` - `Field`* {.preservesOr.} = object - case orKind*: FieldKind - of FieldKind.`int`: - `int`*: int - - of FieldKind.`string`: - `string`*: string + `log`* {.preservesEmbedded.}: Value + `outputs`* {.preservesEmbedded.}: Value + + Derivation* {.preservesRecord: "drv".} = object + `expr`*: string + `storePath`*: string + + RealiseResultKind* {.pure.} = enum + `Error`, `Outputs` + `RealiseResult`* {.preservesOr.} = object + case orKind*: RealiseResultKind + of RealiseResultKind.`Error`: + `error`*: Error + + of RealiseResultKind.`Outputs`: + `outputs`*: Outputs - StringSet* = HashSet[string] - AddToStoreAttrs* {.preservesDictionary.} = object - `ca`*: string - `ca-method`*: Symbol - `deriver`*: string - `eris`*: seq[byte] - `name`*: string - `narHash`*: string - `narSize`*: BiggestInt - `references`*: StringSeq - `registrationTime`*: BiggestInt - `sigs`*: StringSet - `ultimate`*: bool - - AddToStoreClientAttrs* {.preservesDictionary.} = object - `ca-method`*: Symbol - `eris`*: seq[byte] - `name`*: string - `references`*: StringSeq - - PathInfo* {.preservesRecord: "path".} = object - `path`*: string - `attrs`*: AttrSet - - Build* {.preservesRecord: "nix-build".} = object - `input`*: string - `output`*: Preserve[void] - - Fields* = seq[Field] - ActionStart* {.preservesRecord: "start".} = object - `id`*: BiggestInt - `level`*: BiggestInt - `type`*: BiggestInt - `text`*: string - `fields`*: Fields - `parent`*: BiggestInt + EvalSuccess* {.preservesTuple.} = object + `expr`*: string + `result`*: Value + + EvalResultKind* {.pure.} = enum + `Error`, `EvalSuccess` + `EvalResult`* {.preservesOr.} = object + case orKind*: EvalResultKind + of EvalResultKind.`Error`: + `error`*: Error + + of EvalResultKind.`EvalSuccess`: + `evalsuccess`*: EvalSuccess + + + InstantiateResultKind* {.pure.} = enum + `Error`, `Derivation` + `InstantiateResult`* {.preservesOr.} = object + case orKind*: InstantiateResultKind + of InstantiateResultKind.`Error`: + `error`*: Error + + of InstantiateResultKind.`Derivation`: + `derivation`*: Derivation + + + ResolveStep* {.preservesRecord: "nix-actor".} = object + `detail`*: ResolveDetail Instantiate* {.preservesRecord: "instantiate".} = object `expr`*: string + `log`* {.preservesEmbedded.}: Value + `result`*: InstantiateResult + + Outputs* {.preservesRecord: "outputs".} = object + `drv`*: string + `storePaths`*: seq[string] + + ResolveDetail* {.preservesDictionary.} = object + `command-path`*: seq[string] + `lookupPath`*: seq[string] `options`*: AttrSet - `result`*: Preserve[void] - - StringSeq* = seq[string] - ActionStop* {.preservesRecord: "stop".} = object - `id`*: BiggestInt - - ActionResult* {.preservesRecord: "result".} = object - `id`*: BiggestInt - `type`*: BiggestInt - `fields`*: Fields - -proc `$`*(x: Eval | AttrSet | Realise | LegacyPathAttrs | Missing | Narinfo | - Field | - StringSet | - AddToStoreAttrs | - AddToStoreClientAttrs | - PathInfo | - Build | - Fields | - ActionStart | + +proc `$`*(x: Error | Eval | AttrSet | Realise | Derivation | RealiseResult | + EvalSuccess | + EvalResult | + InstantiateResult | + ResolveStep | Instantiate | - StringSeq | - ActionStop | - ActionResult): string = - `$`(toPreserve(x)) - -proc encode*(x: Eval | AttrSet | Realise | LegacyPathAttrs | Missing | Narinfo | - Field | - StringSet | - AddToStoreAttrs | - AddToStoreClientAttrs | - PathInfo | - Build | - Fields | - ActionStart | + Outputs | + ResolveDetail): string = + `$`(toPreserves(x)) + +proc encode*(x: Error | Eval | AttrSet | Realise | Derivation | RealiseResult | + EvalSuccess | + EvalResult | + InstantiateResult | + ResolveStep | Instantiate | - StringSeq | - ActionStop | - ActionResult): seq[byte] = - encode(toPreserve(x)) + Outputs | + ResolveDetail): seq[byte] = + encode(toPreserves(x)) diff --git a/src/nix_actor/sockets.nim b/src/nix_actor/sockets.nim deleted file mode 100644 index 13fc093a5308..000000000000 --- a/src/nix_actor/sockets.nim +++ /dev/null @@ -1,213 +0,0 @@ -# SPDX-FileCopyrightText: ☭ Emery Hemingway -# SPDX-License-Identifier: Unlicense - -## Common module for communicating with Nix clients and daemons. - -import std/[asyncdispatch, asyncnet, sets, strtabs, strutils, tables] -from std/nativesockets import AF_UNIX, SOCK_STREAM, Protocol - -import eris -import preserves, syndicate - -import ./protocol - -{.pragma: workerProtocol, importc, header: "worker-protocol.hh".} - -type Word* = uint64 - -proc `[]=`*[T](attrs: var AttrSet; key: string; val: T) = - attrs[Symbol key] = val.toPreserve - -const - WORKER_MAGIC_1* = 0x6E697863 - WORKER_MAGIC_2* = 0x6478696F - PROTOCOL_VERSION* = 0x100 or 34 - - STDERR_NEXT* = 0x6F6C6d67 - STDERR_READ* = 0x64617461 - STDERR_WRITE* = 0x64617416 - STDERR_LAST* = 0x616C7473 - STDERR_ERROR* = 0x63787470 - STDERR_START_ACTIVITY* = 0x53545254 - STDERR_STOP_ACTIVITY* = 0x53544F50 - STDERR_RESULT* = 0x52534C54 - -type WorkerOperation* = enum - wopInvalid = 0, - wopIsValidPath = 1, - wopHasSubstitutes = 3, - wopQueryPathHash = 4, # obsolete - wopQueryReferences = 5, # obsolete - wopQueryReferrers = 6, - wopAddToStore = 7, - wopAddTextToStore = 8, # obsolete since 1.25, Nix 3.0. Use wopAddToStore - wopBuildPaths = 9, - wopEnsurePath = 10, - wopAddTempRoot = 11, - wopAddIndirectRoot = 12, - wopSyncWithGC = 13, - wopFindRoots = 14, - wopExportPath = 16, # obsolete - wopQueryDeriver = 18, # obsolete - wopSetOptions = 19, - wopCollectGarbage = 20, - wopQuerySubstitutablePathInfo = 21, - wopQueryDerivationOutputs = 22, # obsolete - wopQueryAllValidPaths = 23, - wopQueryFailedPaths = 24, - wopClearFailedPaths = 25, - wopQueryPathInfo = 26, - wopImportPaths = 27, # obsolete - wopQueryDerivationOutputNames = 28, # obsolete - wopQueryPathFromHashPart = 29, - wopQuerySubstitutablePathInfos = 30, - wopQueryValidPaths = 31, - wopQuerySubstitutablePaths = 32, - wopQueryValidDerivers = 33, - wopOptimiseStore = 34, - wopVerifyStore = 35, - wopBuildDerivation = 36, - wopAddSignatures = 37, - wopNarFromPath = 38, - wopAddToStoreNar = 39, - wopQueryMissing = 40, - wopQueryDerivationOutputMap = 41, - wopRegisterDrvOutput = 42, - wopQueryRealisation = 43, - wopAddMultipleToStore = 44, - wopAddBuildLog = 45, - wopBuildPathsWithResults = 46, - -type - ProtocolError* = object of IOError - Version* = uint16 - - Session* = ref object - socket*: AsyncSocket - buffer*: seq[Word] - version*: Version - -func major*(version: Version): uint16 = version and 0xff00 -func minor*(version: Version): uint16 = version and 0x00ff - -proc close*(session: Session) = - close(session.socket) - reset(session.buffer) - -proc send*(session: Session; words: varargs[Word]): Future[void] = - if session.buffer.len < words.len: - session.buffer.setLen(words.len) - for i, word in words: session.buffer[i] = word - send(session.socket, addr session.buffer[0], words.len shl 3) - -proc send*(session: Session; s: string): Future[void] = - let wordCount = 1 + ((s.len + 7) shr 3) - if session.buffer.len < wordCount: setLen(session.buffer, wordCount) - session.buffer[0] = Word s.len - if s != "": - session.buffer[pred wordCount] = 0x00 - copyMem(addr session.buffer[1], unsafeAddr s[0], s.len) - send(session.socket, addr session.buffer[0], wordCount shl 3) - -proc send*(session: Session; ss: StringSeq|StringSet): Future[void] = - ## Send a set of strings. The set is sent as a contiguous buffer. - session.buffer[0] = Word ss.len - var off = 1 - for s in ss: - let - stringWordLen = (s.len + 7) shr 3 - bufferWordLen = off+1+stringWordLen - if session.buffer.len < bufferWordLen: - setLen(session.buffer, bufferWordLen) - session.buffer[off] = Word s.len - session.buffer[off+stringWordLen] = 0 # clear the aligning bits - inc(off) - copyMem(addr session.buffer[off], unsafeAddr s[0], s.len) - inc(off, stringWordLen) - send(session.socket, addr session.buffer[0], off shl 3) - -proc recvWord*(sock: AsyncSocket): Future[Word] {.async.} = - var w: Word - let n = await recvInto(sock, addr w, sizeof(Word)) - if n != sizeof(Word): raise newException(ProtocolError, "short read") - return w - -proc recvWord*(session: Session): Future[Word] = - recvWord(session.socket) - -proc discardWords*(session: Session; n: int): Future[void] {.async.} = - if session.buffer.len < n: setLen(session.buffer, n) - let byteCount = n shl 3 - let n = await recvInto(session.socket, addr session.buffer[0], byteCount) - if n != byteCount: - raise newException(ProtocolError, "short read") - -proc recvString*(socket: AsyncSocket): Future[string] {.async.} = - let stringLen = int (await recvWord(socket)) - if stringLen > 0: - var s = newString((stringLen + 7) and (not 7)) - let n = await recvInto(socket, addr s[0], s.len) - if n != s.len: - raise newException(ProtocolError, "short read") - setLen(s, stringLen) - return s - return "" - -proc recvString*(session: Session): Future[string] = - recvString(session.socket) - -proc recvStringSeq*(session: Session): Future[StringSeq] {.async.} = - let count = int(await recvWord(session.socket)) - var strings = newSeq[string](count) - for i in 0.. Date: Tue, 25 Jun 2024 12:29:34 +0300 Subject: README: move example snippet to a working script --- README.md | 60 ++---------------------------------------------------------- sbom.json | 2 +- test.pr | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 59 deletions(-) create mode 100755 test.pr diff --git a/README.md b/README.md index d3b7b753672c..e1166968e868 100644 --- a/README.md +++ b/README.md @@ -4,62 +4,6 @@ An actor for interacting with the [Nix](https://nixos.org/) daemon via the [Synd See [protocol.prs](./protocol.prs) for the Syndicate protocol [schema](https://preserves.dev/preserves-schema.html). -The was once an abstraction of the Nix worker socket that could intermediate between clients and the worker but that code has been removed, refer to git history for that. - -*This is only a proof-of-concept and is probably not useful yet.* - -## Example configuration - -A demo script for the [Syndicate server](https://git.syndicate-lang.org/syndicate-lang/syndicate-rs), see https://synit.org/book/operation/scripting.html -``` -#!/usr/bin/env -S syndicate-server --config - -let ?nixLog = dataspace -$nixLog ?? ?line [ - # Log out to the syndicate-server log. - $log ! -] - -let ?results = dataspace -$results ? ?any [ - # Just log everything else. - $log ! -] - -let ?resolver = dataspace -$resolver ? $cap [ +For an example configuration see [test.pr](./test.pr). - - - {}; in pkgs.cowsay" $nixLog $results> - - $results ? [ - $log ! - $cap - ] - -] - -? ?cap> [ - $cap $resolver> -] - -> - - -``` +The was once an abstraction of the Nix worker socket that could intermediate between clients and the worker but that code has been removed, refer to git history for that. diff --git a/sbom.json b/sbom.json index f503d24f36f8..fdc52a375286 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240624", + "version": "20240625", "authors": [ { "name": "Emery Hemingway" diff --git a/test.pr b/test.pr new file mode 100755 index 000000000000..cd9b3858241d --- /dev/null +++ b/test.pr @@ -0,0 +1,51 @@ +#!/usr/bin/env -S syndicate-server --control --config + +let ?nixLog = dataspace +$nixLog ?? ?line [ + $log ! +] + +let ?results = dataspace +$results ? ?any [ + $log ! +] + +? $cap [ + + + + {}; in pkgs.cowsay" $nixLog $results> + + # Realise all observed store derivations. + $results ? [ + $cap + ] + +] + +let ?resolver = <* $config [ >]> +? ?cap> [ + $cap $resolver> +] + +> + +> +? [ + +] -- cgit 1.4.1 From ddc523751ade7c03d0e1c881a6d4ebe15aec64a8 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 25 Jun 2024 12:29:59 +0300 Subject: Use $PKG_CONFIG instead of pkg-config This is to abstract pkg-config for cross-compilation. --- src/nix_actor/libnix/libexpr.nim | 4 ++-- src/nix_actor/libnix/main.nim | 4 ++-- src/nix_actor/libnix/store.nim | 4 ++-- src/nix_actor/nix_api.nim | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/nix_actor/libnix/libexpr.nim b/src/nix_actor/libnix/libexpr.nim index d98e20052990..cd5722bb0505 100644 --- a/src/nix_actor/libnix/libexpr.nim +++ b/src/nix_actor/libnix/libexpr.nim @@ -3,8 +3,8 @@ import ./stdpuspus, ./store -{.passC: staticExec("pkg-config --cflags nix-expr").} -{.passL: staticExec("pkg-config --libs nix-expr").} +{.passC: staticExec"$PKG_CONFIG --cflags nix-expr".} +{.passL: staticExec"$PKG_CONFIG --libs nix-expr".} proc parentDir(path: string): string = var i = path.high diff --git a/src/nix_actor/libnix/main.nim b/src/nix_actor/libnix/main.nim index 072e97e589e5..0d38d41dab59 100644 --- a/src/nix_actor/libnix/main.nim +++ b/src/nix_actor/libnix/main.nim @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -{.passC: staticExec("pkg-config --cflags nix-main").} -{.passL: staticExec("pkg-config --libs nix-main").} +{.passC: staticExec"$PKG_CONFIG --cflags nix-main".} +{.passL: staticExec"$PKG_CONFIG --libs nix-main".} proc initNix*() {.importcpp: "nix::initNix", header: "shared.hh".} diff --git a/src/nix_actor/libnix/store.nim b/src/nix_actor/libnix/store.nim index a159927cdd6b..b8b890397a7b 100644 --- a/src/nix_actor/libnix/store.nim +++ b/src/nix_actor/libnix/store.nim @@ -1,8 +1,8 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -{.passC: staticExec("pkg-config --cflags nix-store").} -{.passL: staticExec("pkg-config --libs nix-store").} +{.passC: staticExec"$PKG_CONFIG --cflags nix-store".} +{.passL: staticExec"$PKG_CONFIG --libs nix-store".} {.passC: "'-DSYSTEM=\"x86_64-linux\"'".} diff --git a/src/nix_actor/nix_api.nim b/src/nix_actor/nix_api.nim index 5486b32cb1ed..d4700ef2da55 100644 --- a/src/nix_actor/nix_api.nim +++ b/src/nix_actor/nix_api.nim @@ -6,8 +6,8 @@ import ./nix_api_types, ./nix_api_value, ./nix_api_store, ./nix_api_expr export NixContext, Store, EvalState, Value, ValueType, gc_decref, isNil -{.passC: staticExec("pkg-config --cflags nix-expr-c").} -{.passL: staticExec("pkg-config --libs nix-expr-c").} +{.passC: staticExec"$PKG_CONFIG --cflags nix-expr-c".} +{.passL: staticExec"$PKG_CONFIG --libs nix-expr-c".} # Always pass NixContext as a nil pointer. -- cgit 1.4.1 From 1fcd050fa98a88d9775ae7e4f03980760c65edfb Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Wed, 26 Jun 2024 19:56:49 +0300 Subject: Remove log field from eval Not known if trace messages from evaluation can be captured. --- protocol.prs | 2 +- sbom.json | 16 ++++++++-------- src/nix_actor.nim | 2 +- src/nix_actor/protocol.nim | 3 +-- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/protocol.prs b/protocol.prs index 88bd54c3a42c..a2eea711ce39 100644 --- a/protocol.prs +++ b/protocol.prs @@ -24,7 +24,7 @@ Error = . # Asserted to nix-actor. # @expr is evaluated and asserted to @result # with log lines messaged to @log. -Eval = . +Eval = . EvalResult = Error / EvalSuccess . EvalSuccess = [@expr string @result any] . diff --git a/sbom.json b/sbom.json index fdc52a375286..b2cc3cdf7981 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240625", + "version": "20240626", "authors": [ { "name": "Emery Hemingway" @@ -129,10 +129,10 @@ "type": "library", "bom-ref": "pkg:nim/preserves", "name": "preserves", - "version": "20240623", + "version": "trunk", "externalReferences": [ { - "url": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/19f5641ec0459d60f0ae992aa72151c45a73ef2f.tar.gz", + "url": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/2e48681eb43c28737d08b793c3ad48f18379dc5f.tar.gz", "type": "source-distribution" }, { @@ -147,23 +147,23 @@ }, { "name": "nix:fod:path", - "value": "/nix/store/l2xg6y34x7h9sxbx8fk61j9m4ramqaac-source" + "value": "/nix/store/v9140vbdxz9va8l8qsqyra0dcfgfw9dc-source" }, { "name": "nix:fod:rev", - "value": "19f5641ec0459d60f0ae992aa72151c45a73ef2f" + "value": "2e48681eb43c28737d08b793c3ad48f18379dc5f" }, { "name": "nix:fod:sha256", - "value": "08q86d7s96wy8190z2973bb0cjp5k6xdakna1669672m1davqxnl" + "value": "0mgwdi9qcpsjpylds3nn7j88qal724hxhkamv949h3g01zyy038b" }, { "name": "nix:fod:url", - "value": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/19f5641ec0459d60f0ae992aa72151c45a73ef2f.tar.gz" + "value": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/2e48681eb43c28737d08b793c3ad48f18379dc5f.tar.gz" }, { "name": "nix:fod:ref", - "value": "20240623" + "value": "trunk" }, { "name": "nix:fod:srcDir", diff --git a/src/nix_actor.nim b/src/nix_actor.nim index dc9f392f39bc..87775796c457 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -169,7 +169,7 @@ proc eval(store: Store; state: EvalState; expr: string): EvalResult = close(nixVal) proc serve(turn: Turn; detail: ResolveDetail; store: Store; state: EvalState; ds: Cap) = - during(turn, ds, Eval.grabWithin) do (expr: string, log: Cap, resp: Cap): + during(turn, ds, Eval.grabWithin) do (expr: string, resp: Cap): discard publish(turn, resp, eval(store, state, expr)) during(turn, ds, Instantiate.grabWithin) do (expr: string, log: Value, resp: Cap): diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index caa1330390ab..835c835bc002 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -8,8 +8,7 @@ type Eval* {.preservesRecord: "eval".} = object `expr`*: string - `log`* {.preservesEmbedded.}: Value - `result`*: EvalResult + `result`* {.preservesEmbedded.}: Value AttrSet* = Table[Symbol, Value] Realise* {.preservesRecord: "realise".} = object -- cgit 1.4.1 From a6baa76fdfe0593986a3fc4a44f965a7e18f2c46 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Wed, 26 Jun 2024 19:57:45 +0300 Subject: Add interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Evaluating Preserves values against files containing Nix code makes it possible to evaluate arbitrary Nix code with somewhat arbitrary Preserves values. --- protocol.prs | 3 +++ src/nix_actor.nim | 24 ++++++++++++++++++++++++ src/nix_actor/protocol.nim | 27 +++++++++++++++++++++++++++ test.pr | 4 +++- 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/protocol.prs b/protocol.prs index a2eea711ce39..2bcc3a07d3f1 100644 --- a/protocol.prs +++ b/protocol.prs @@ -25,8 +25,11 @@ Error = . # @expr is evaluated and asserted to @result # with log lines messaged to @log. Eval = . +EvalFile = . EvalResult = Error / EvalSuccess . EvalSuccess = [@expr string @result any] . +EvalFileResult = Error / @success EvalFileSuccess . +EvalFileSuccess = [@path string @args any @result any] . # Asserted to nix-actor. # @expr is instantiated to a store-derivation which is asserted to @result. diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 87775796c457..99600b2c27e0 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -168,10 +168,34 @@ proc eval(store: Store; state: EvalState; expr: string): EvalResult = finally: close(nixVal) +proc evalFile(store: Store; state: EvalState; path: string; args: Value): EvalFileResult = + var + js = args.jsonText + stderr.writeLine "converted ", $args, " to ", js + var + expr = """import $1 (builtins.fromJSON ''$2'')""" % [ path, js ] + # TODO: convert to NixValue instead of using JSON conversion. + nixVal: NixValue + try: + nixVal = state.evalFromString(expr, "") + state.force(nixVal) + result = EvalFileResult(orKind: EvalFileResultKind.success) + result.success.path = path + result.success.args = args + result.success.result = nixVal.toPreserves(state).mapEmbeds(unthunk) + except CatchableError as err: + reset result + result.error.message = err.msg + finally: + close(nixVal) + proc serve(turn: Turn; detail: ResolveDetail; store: Store; state: EvalState; ds: Cap) = during(turn, ds, Eval.grabWithin) do (expr: string, resp: Cap): discard publish(turn, resp, eval(store, state, expr)) + during(turn, ds, EvalFile.grabWithin) do (path: string, args: Value, resp: Cap): + discard publish(turn, resp, evalFile(store, state, path, args)) + during(turn, ds, Instantiate.grabWithin) do (expr: string, log: Value, resp: Cap): discard publish(turn, resp, instantiate(turn.facet, detail, expr, log.unembed(Cap))) diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 835c835bc002..9ce51afeb48c 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -35,6 +35,11 @@ type `expr`*: string `result`*: Value + EvalFile* {.preservesRecord: "eval-file".} = object + `path`*: string + `args`*: Value + `result`* {.preservesEmbedded.}: Value + EvalResultKind* {.pure.} = enum `Error`, `EvalSuccess` `EvalResult`* {.preservesOr.} = object @@ -60,11 +65,27 @@ type ResolveStep* {.preservesRecord: "nix-actor".} = object `detail`*: ResolveDetail + EvalFileResultKind* {.pure.} = enum + `Error`, `success` + `EvalFileResult`* {.preservesOr.} = object + case orKind*: EvalFileResultKind + of EvalFileResultKind.`Error`: + `error`*: Error + + of EvalFileResultKind.`success`: + `success`*: EvalFileSuccess + + Instantiate* {.preservesRecord: "instantiate".} = object `expr`*: string `log`* {.preservesEmbedded.}: Value `result`*: InstantiateResult + EvalFileSuccess* {.preservesTuple.} = object + `path`*: string + `args`*: Value + `result`*: Value + Outputs* {.preservesRecord: "outputs".} = object `drv`*: string `storePaths`*: seq[string] @@ -76,20 +97,26 @@ type proc `$`*(x: Error | Eval | AttrSet | Realise | Derivation | RealiseResult | EvalSuccess | + EvalFile | EvalResult | InstantiateResult | ResolveStep | + EvalFileResult | Instantiate | + EvalFileSuccess | Outputs | ResolveDetail): string = `$`(toPreserves(x)) proc encode*(x: Error | Eval | AttrSet | Realise | Derivation | RealiseResult | EvalSuccess | + EvalFile | EvalResult | InstantiateResult | ResolveStep | + EvalFileResult | Instantiate | + EvalFileSuccess | Outputs | ResolveDetail): seq[byte] = encode(toPreserves(x)) diff --git a/test.pr b/test.pr index cd9b3858241d..e4f6dfb6a6db 100755 --- a/test.pr +++ b/test.pr @@ -12,7 +12,9 @@ $results ? ?any [ ? $cap [ - + + + {}; in pkgs.cowsay" $nixLog $results> -- cgit 1.4.1 From 9abc4dab8b66f6bf9ad22ae289f31d0f9f330225 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Fri, 28 Jun 2024 19:25:20 +0300 Subject: Update Preserves and Syndicate dependencies --- sbom.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/sbom.json b/sbom.json index b2cc3cdf7981..eb4fdad0b5b2 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240626", + "version": "20240628", "authors": [ { "name": "Emery Hemingway" @@ -45,10 +45,10 @@ "type": "library", "bom-ref": "pkg:nim/syndicate", "name": "syndicate", - "version": "20240623", + "version": "trunk", "externalReferences": [ { - "url": "https://git.syndicate-lang.org/ehmry/syndicate-nim/archive/cdc59b76415730b46107a842b79c31d3ecbf48ea.tar.gz", + "url": "https://git.syndicate-lang.org/ehmry/syndicate-nim/archive/a2c723914b200e44e64ea67f4bf3a57b5d7e8a04.tar.gz", "type": "source-distribution" }, { @@ -63,23 +63,23 @@ }, { "name": "nix:fod:path", - "value": "/nix/store/pqwf2h9394hpp7195h977bv5zdnvf6z5-source" + "value": "/nix/store/vd1v0mdi7ww2rgy1s4s1hy3ll633lh4c-source" }, { "name": "nix:fod:rev", - "value": "cdc59b76415730b46107a842b79c31d3ecbf48ea" + "value": "a2c723914b200e44e64ea67f4bf3a57b5d7e8a04" }, { "name": "nix:fod:sha256", - "value": "1cj31y9y3ibq5sa08v01i0fccmw1lr52v70x8brs44mr8ckc9bfp" + "value": "04f6rpz22xi308pg80lplsnpgg1l3lxwrj86krj4mng91kv8gs3c" }, { "name": "nix:fod:url", - "value": "https://git.syndicate-lang.org/ehmry/syndicate-nim/archive/cdc59b76415730b46107a842b79c31d3ecbf48ea.tar.gz" + "value": "https://git.syndicate-lang.org/ehmry/syndicate-nim/archive/a2c723914b200e44e64ea67f4bf3a57b5d7e8a04.tar.gz" }, { "name": "nix:fod:ref", - "value": "20240623" + "value": "trunk" }, { "name": "nix:fod:srcDir", @@ -132,7 +132,7 @@ "version": "trunk", "externalReferences": [ { - "url": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/2e48681eb43c28737d08b793c3ad48f18379dc5f.tar.gz", + "url": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/e22b354c11c6b9a87e3cde0e8958daa1414fb05b.tar.gz", "type": "source-distribution" }, { @@ -147,19 +147,19 @@ }, { "name": "nix:fod:path", - "value": "/nix/store/v9140vbdxz9va8l8qsqyra0dcfgfw9dc-source" + "value": "/nix/store/lim0cw9vzy8pr31y3mvsxnc5xlbf6rnz-source" }, { "name": "nix:fod:rev", - "value": "2e48681eb43c28737d08b793c3ad48f18379dc5f" + "value": "e22b354c11c6b9a87e3cde0e8958daa1414fb05b" }, { "name": "nix:fod:sha256", - "value": "0mgwdi9qcpsjpylds3nn7j88qal724hxhkamv949h3g01zyy038b" + "value": "1bskicgm7bx7sk0fig58h98xznzlnc0wqfbskhh0b5jsyzjrqi60" }, { "name": "nix:fod:url", - "value": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/2e48681eb43c28737d08b793c3ad48f18379dc5f.tar.gz" + "value": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/e22b354c11c6b9a87e3cde0e8958daa1414fb05b.tar.gz" }, { "name": "nix:fod:ref", -- cgit 1.4.1 From bddd7a36a196b139b8209312c7dfb9af6f8e7484 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Fri, 28 Jun 2024 19:25:52 +0300 Subject: Move Nix data conversion to sub-module --- sbom.json | 2 +- src/nix_actor.nim | 66 ++----------------------------------- src/nix_actor/nix_api_value.nim | 2 +- src/nix_actor/nix_values.nim | 72 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 65 deletions(-) create mode 100644 src/nix_actor/nix_values.nim diff --git a/sbom.json b/sbom.json index eb4fdad0b5b2..478ef9fbdce0 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240628", + "version": "20240629", "authors": [ { "name": "Emery Hemingway" diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 99600b2c27e0..0f7f3ee70878 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -7,7 +7,7 @@ import pkg/syndicate, pkg/syndicate/protocols/gatekeeper, pkg/syndicate/relays, - ./nix_actor/[nix_api, nix_api_util, nix_api_value], + ./nix_actor/[nix_api, nix_values], ./nix_actor/protocol proc echo(args: varargs[string, `$`]) {.used.} = @@ -15,63 +15,6 @@ proc echo(args: varargs[string, `$`]) {.used.} = type Value = preserves.Value - NixValue = nix_api.Value - StringThunkRef = ref StringThunkObj - StringThunkObj = object of EmbeddedObj - data: Option[string] - -proc thunkString(start: cstring; n: cuint; state: pointer) {.cdecl.} = - let thunk = cast[ptr StringThunkObj](state) - assert thunk.data.isNone - var buf = newString(n) - copyMem(buf[0].addr, start, buf.len) - thunk.data = buf.move.some - -proc unthunk(v: Value): Value = - let thunk = v.unembed(StringThunkRef) - assert thunk.isSome - assert thunk.get.data.isSome - thunk.get.data.get.toPreserves - -proc toPreserves(value: NixValue; state: EvalState): Value {.gcsafe.} = - var ctx: NixContext - case get_type(ctx, value) - of NIX_TYPE_THUNK: raiseAssert "cannot preserve thunk" - of NIX_TYPE_INT: - result = getInt(ctx, value).toPreserves - of NIX_TYPE_FLOAT: - result = getFloat(ctx, value).toPreserves - of NIX_TYPE_BOOL: - result = getBool(ctx, value).toPreserves - of NIX_TYPE_STRING: - let thunk = StringThunkRef() - let err = getString(ctx, value, thunkString, thunk[].addr) - doAssert err == NIX_OK, $err - result = thunk.embed - of NIX_TYPE_PATH: - result = ($getPathString(ctx, value)).toPreserves - of NIX_TYPE_NULL: - result = initRecord("null") - of NIX_TYPE_ATTRS: - result = initDictionary() - let n = getAttrsSize(ctx, value) - var i: cuint - while i < n: - var (key, val) = get_attr_byidx(ctx, value, state, i) - inc(i) - result[toSymbol($key)] = val.toPreserves(state) - # close(val) - of NIX_TYPE_LIST: - let n = getListSize(ctx, value) - result = initSequence(n) - var i: cuint - while i < n: - var val = getListByIdx(ctx, value, state, i) - result[i] = val.toPreserves(state) - inc(i) - # close(val) - of NIX_TYPE_FUNCTION, NIX_TYPE_EXTERNAL: - raiseAssert "TODO: need a failure type" proc findCommand(detail: ResolveDetail; cmd: string): string = for dir in detail.`command-path`: @@ -170,13 +113,10 @@ proc eval(store: Store; state: EvalState; expr: string): EvalResult = proc evalFile(store: Store; state: EvalState; path: string; args: Value): EvalFileResult = var - js = args.jsonText - stderr.writeLine "converted ", $args, " to ", js - var - expr = """import $1 (builtins.fromJSON ''$2'')""" % [ path, js ] - # TODO: convert to NixValue instead of using JSON conversion. nixVal: NixValue try: + var expr = """import $1 (builtins.fromJSON ''$2'')""" % [ path, args.jsonText ] + # TODO: convert to NixValue instead of using JSON conversion. nixVal = state.evalFromString(expr, "") state.force(nixVal) result = EvalFileResult(orKind: EvalFileResultKind.success) diff --git a/src/nix_actor/nix_api_value.nim b/src/nix_actor/nix_api_value.nim index a93a981db3a2..bed607a0f8c0 100644 --- a/src/nix_actor/nix_api_value.nim +++ b/src/nix_actor/nix_api_value.nim @@ -41,7 +41,7 @@ proc has_attr_byname*(context: NixContext; value: Value; state: EvalState; name: proc get_attr_byidx*(context: NixContext; value: Value; state: EvalState; i: cuint; name: ptr cstring): Value {.nix_api_value.} -proc get_attr_name_byidx*(context: NixContext; value: Value; state: EvalState; i: cuint): cstring {.nix_api_value.} +# proc get_attr_name_byidx*(context: NixContext; value: Value; state: EvalState; i: cuint): cstring {.nix_api_value.} proc init_bool*(context: NixContext; value: Value; b: bool): nix_err {.nix_api_value.} diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim new file mode 100644 index 000000000000..7268dcd48b58 --- /dev/null +++ b/src/nix_actor/nix_values.nim @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +import + std/options, + pkg/preserves, + ./[nix_api, nix_api_util, nix_api_value] + +proc echo(args: varargs[string, `$`]) {.used.} = + stderr.writeLine(args) + +type + Value = preserves.Value + NixValue* = nix_api.Value + StringThunkRef = ref StringThunkObj + StringThunkObj = object of EmbeddedObj + data: Option[string] + +proc thunkString(start: cstring; n: cuint; state: pointer) {.cdecl.} = + let thunk = cast[ptr StringThunkObj](state) + assert thunk.data.isNone + var buf = newString(n) + if n > 0: + copyMem(buf[0].addr, start, buf.len) + thunk.data = buf.move.some + +proc unthunk*(v: Value): Value = + let thunk = v.unembed(StringThunkRef) + assert thunk.isSome + assert thunk.get.data.isSome + thunk.get.data.get.toPreserves + +proc toPreserves*(value: NixValue; state: EvalState): Value {.gcsafe.} = + var ctx: NixContext + let kind = get_type(ctx, value) + case kind + of NIX_TYPE_THUNK: raiseAssert "cannot preserve thunk" + of NIX_TYPE_INT: + result = getInt(ctx, value).toPreserves + of NIX_TYPE_FLOAT: + result = getFloat(ctx, value).toPreserves + of NIX_TYPE_BOOL: + result = getBool(ctx, value).toPreserves + of NIX_TYPE_STRING: + let thunk = StringThunkRef() + let err = getString(ctx, value, thunkString, thunk[].addr) + doAssert err == NIX_OK, $err + result = thunk.embed + of NIX_TYPE_PATH: + result = ($getPathString(ctx, value)).toPreserves + of NIX_TYPE_NULL: + result = initRecord("null") + of NIX_TYPE_ATTRS: + let n = getAttrsSize(ctx, value) + result = initDictionary(int n) + var i: cuint + while i < n: + let (key, val) = get_attr_byidx(ctx, value, state, i) + result[($key).toSymbol] = val.toPreserves(state) + inc(i) + of NIX_TYPE_LIST: + let n = getListSize(ctx, value) + result = initSequence(n) + var i: cuint + while i < n: + var val = getListByIdx(ctx, value, state, i) + result[i] = val.toPreserves(state) + inc(i) + of NIX_TYPE_FUNCTION: + result = "«function»".toPreserves + of NIX_TYPE_EXTERNAL: + result = "«external»".toPreserves -- cgit 1.4.1 From 3acfc33897cbd47ece616e09e966c2cf765253c5 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Fri, 28 Jun 2024 19:27:26 +0300 Subject: Reduce derivation values to outPath strings The derivation builtin produces an attribute set that can often only be lazily evaluated. --- src/nix_actor/nix_values.nim | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim index 7268dcd48b58..e608033ea72c 100644 --- a/src/nix_actor/nix_values.nim +++ b/src/nix_actor/nix_values.nim @@ -51,13 +51,16 @@ proc toPreserves*(value: NixValue; state: EvalState): Value {.gcsafe.} = of NIX_TYPE_NULL: result = initRecord("null") of NIX_TYPE_ATTRS: - let n = getAttrsSize(ctx, value) - result = initDictionary(int n) - var i: cuint - while i < n: - let (key, val) = get_attr_byidx(ctx, value, state, i) - result[($key).toSymbol] = val.toPreserves(state) - inc(i) + if has_attr_byname(ctx, value, state, "outPath"): + result = get_attr_byname(ctx, value, state, "outPath").toPreserves(state) + else: + let n = getAttrsSize(ctx, value) + result = initDictionary(int n) + var i: cuint + while i < n: + let (key, val) = get_attr_byidx(ctx, value, state, i) + result[($key).toSymbol] = val.toPreserves(state) + inc(i) of NIX_TYPE_LIST: let n = getListSize(ctx, value) result = initSequence(n) -- cgit 1.4.1 From 7e8870051623d78f918b7e15bf15a185b1ce2dbd Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sat, 29 Jun 2024 23:25:20 +0300 Subject: Remove instantiation, rewrite protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit and file can return records that are already instantiated. Use the / pattern for results. Assert each realised store path individually rather than as a list. --- protocol.prs | 37 +++++++++++++---------- src/nix_actor.nim | 69 +++++++++++------------------------------- src/nix_actor/nix_values.nim | 10 ++++-- src/nix_actor/protocol.nim | 72 ++++++++++++++++---------------------------- test.pr | 47 ++++++++++++++++------------- 5 files changed, 99 insertions(+), 136 deletions(-) diff --git a/protocol.prs b/protocol.prs index 2bcc3a07d3f1..903ba55002ec 100644 --- a/protocol.prs +++ b/protocol.prs @@ -22,26 +22,31 @@ ResolveDetail = { Error = . # Asserted to nix-actor. -# @expr is evaluated and asserted to @result -# with log lines messaged to @log. +# @expr is evaluated and asserted to @result. Eval = . -EvalFile = . -EvalResult = Error / EvalSuccess . -EvalSuccess = [@expr string @result any] . -EvalFileResult = Error / @success EvalFileSuccess . -EvalFileSuccess = [@path string @args any @result any] . +EvalResult = @err Error / @ok EvalSuccess . +EvalSuccess = . # Asserted to nix-actor. -# @expr is instantiated to a store-derivation which is asserted to @result. -Instantiate = . -InstantiateResult = Error / Derivation . -Derivation = . +# @file is imported as a function, @args are applied, and the result asserted +# to @result. +EvalFile = . +EvalFileResult = @err Error / @ok EvalFileSuccess . +EvalFileSuccess = . + +# Represents a Nix derivation. +# The @storePath can be realized as store object from @drvPath. +# +# If an attrset value resulting from evaluation has a "drvPath" attribute +# then a drv record is returned in place of the attrset. Returning the +# attrset is not feasible because otherwise lazy values would explode into +# complete dependency trees. +Derivation = . # Asserted to nix-actor. -# The list of store-paths from realising @drv -# are asserted to @outputs as a list of strings. -Realise = . -RealiseResult = Error / Outputs . -Outputs = . +# @drvPath is realised and each output is asserted to @outputs. +Realise = . +RealiseResult = Error / RealiseSuccess . +RealiseSuccess = . AttrSet = {symbol: any ...:...} . diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 0f7f3ee70878..5c03e0c16071 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -35,38 +35,7 @@ proc commandlineArgs(detail: ResolveDetail; args: varargs[string]): seq[string] proc commandlineEnv(detail: ResolveDetail): StringTableRef = newStringTable({"NIX_PATH": detail.lookupPath.join ":"}) -proc instantiate(facet: Facet; detail: ResolveDetail; expr: string; log: Option[Cap]): InstantiateResult = - # TODO: run this process asynchronously with nim-sys. - var p: Process - try: - p = startProcess( - detail.findCommand("nix-instantiate"), - args = detail.commandlineArgs("--expr", expr), - env = detail.commandlineEnv(), - options = {}, - ) - var - errors = errorStream(p) - line = "".toPreserves - while true: - if errors.readLine(line.string): - if log.isSome: - facet.run do (turn: Turn): - message(turn, log.get, line) - elif not running(p): break - initDuration(milliseconds = 250).some.runOnce - var path = p.outputStream.readAll.strip - if path != "": - result = InstantiateResult(orKind: InstantiateResultKind.Derivation) - result.derivation.expr = expr - result.derivation.storePath = path - except CatchableError as err: - reset result - result.error.message = err.msg - finally: - close(p) - -proc realise(facet: Facet; detail: ResolveDetail; drv: string; log: Option[Cap]): RealiseResult = +proc realise(facet: Facet; detail: ResolveDetail; drv: string; log: Option[Cap], resp: Cap) = # TODO: run this process asynchronously with nim-sys. var p: Process try: @@ -87,13 +56,14 @@ proc realise(facet: Facet; detail: ResolveDetail; drv: string; log: Option[Cap]) elif not running(p): break initDuration(milliseconds = 250).some.runOnce var storePaths = p.outputStream.readAll.strip.split - if storePaths != @[]: - result = RealiseResult(orKind: RealiseResultKind.Outputs) - result.outputs.drv = drv - result.outputs.storePaths = storePaths + doAssert storePaths != @[] + facet.run do (turn: Turn): + for path in storePaths: + discard publish(turn, resp, RealiseSuccess( + storePath: path, drvPath: drv)) except CatchableError as err: - reset result - result.error.message = err.msg + facet.run do (turn: Turn): + discard publish(turn, resp, Error(message: err.msg)) finally: close(p) @@ -102,12 +72,12 @@ proc eval(store: Store; state: EvalState; expr: string): EvalResult = try: nixVal = state.evalFromString(expr, "") state.force(nixVal) - result = EvalResult(orKind: EvalResultKind.EvalSuccess) - result.evalsuccess.expr = expr - result.evalsuccess.result = nixVal.toPreserves(state).mapEmbeds(unthunk) + result = EvalResult(orKind: EvalResultKind.ok) + result.ok.result = nixVal.toPreserves(state).unthunkAll + result.ok.expr = expr except CatchableError as err: reset result - result.error.message = err.msg + result.err.message = err.msg finally: close(nixVal) @@ -119,13 +89,13 @@ proc evalFile(store: Store; state: EvalState; path: string; args: Value): EvalFi # TODO: convert to NixValue instead of using JSON conversion. nixVal = state.evalFromString(expr, "") state.force(nixVal) - result = EvalFileResult(orKind: EvalFileResultKind.success) - result.success.path = path - result.success.args = args - result.success.result = nixVal.toPreserves(state).mapEmbeds(unthunk) + result = EvalFileResult(orKind: EvalFileResultKind.ok) + result.ok.result = nixVal.toPreserves(state).unthunkAll + result.ok.args = args + result.ok.path = path except CatchableError as err: reset result - result.error.message = err.msg + result.err.message = err.msg finally: close(nixVal) @@ -136,11 +106,8 @@ proc serve(turn: Turn; detail: ResolveDetail; store: Store; state: EvalState; ds during(turn, ds, EvalFile.grabWithin) do (path: string, args: Value, resp: Cap): discard publish(turn, resp, evalFile(store, state, path, args)) - during(turn, ds, Instantiate.grabWithin) do (expr: string, log: Value, resp: Cap): - discard publish(turn, resp, instantiate(turn.facet, detail, expr, log.unembed(Cap))) - during(turn, ds, Realise.grabWithin) do (drv: string, log: Value, resp: Cap): - discard publish(turn, resp, realise(turn.facet, detail, drv, log.unembed(Cap))) + realise(turn.facet, detail, drv, log.unembed(Cap), resp) proc main() = initLibexpr() diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim index e608033ea72c..5b426dd7d9ab 100644 --- a/src/nix_actor/nix_values.nim +++ b/src/nix_actor/nix_values.nim @@ -30,6 +30,9 @@ proc unthunk*(v: Value): Value = assert thunk.get.data.isSome thunk.get.data.get.toPreserves +proc unthunkAll*(v: Value): Value = + v.mapEmbeds(unthunk) + proc toPreserves*(value: NixValue; state: EvalState): Value {.gcsafe.} = var ctx: NixContext let kind = get_type(ctx, value) @@ -51,8 +54,11 @@ proc toPreserves*(value: NixValue; state: EvalState): Value {.gcsafe.} = of NIX_TYPE_NULL: result = initRecord("null") of NIX_TYPE_ATTRS: - if has_attr_byname(ctx, value, state, "outPath"): - result = get_attr_byname(ctx, value, state, "outPath").toPreserves(state) + if has_attr_byname(ctx, value, state, "drvPath"): + result = initRecord("drv", + get_attr_byname(ctx, value, state, "drvPath").toPreserves(state), + get_attr_byname(ctx, value, state, "outPath").toPreserves(state), + ) else: let n = getAttrsSize(ctx, value) result = initDictionary(int n) diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 9ce51afeb48c..5635440ad6d8 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -12,28 +12,32 @@ type AttrSet* = Table[Symbol, Value] Realise* {.preservesRecord: "realise".} = object - `drv`*: string + `drvPath`*: string `log`* {.preservesEmbedded.}: Value `outputs`* {.preservesEmbedded.}: Value Derivation* {.preservesRecord: "drv".} = object - `expr`*: string + `drvPath`*: string `storePath`*: string RealiseResultKind* {.pure.} = enum - `Error`, `Outputs` + `Error`, `RealiseSuccess` `RealiseResult`* {.preservesOr.} = object case orKind*: RealiseResultKind of RealiseResultKind.`Error`: `error`*: Error - of RealiseResultKind.`Outputs`: - `outputs`*: Outputs + of RealiseResultKind.`RealiseSuccess`: + `realisesuccess`*: RealiseSuccess - EvalSuccess* {.preservesTuple.} = object - `expr`*: string + EvalSuccess* {.preservesRecord: "ok".} = object `result`*: Value + `expr`*: string + + RealiseSuccess* {.preservesRecord: "ok".} = object + `storePath`*: string + `drvPath`*: string EvalFile* {.preservesRecord: "eval-file".} = object `path`*: string @@ -41,54 +45,34 @@ type `result`* {.preservesEmbedded.}: Value EvalResultKind* {.pure.} = enum - `Error`, `EvalSuccess` + `err`, `ok` `EvalResult`* {.preservesOr.} = object case orKind*: EvalResultKind - of EvalResultKind.`Error`: - `error`*: Error - - of EvalResultKind.`EvalSuccess`: - `evalsuccess`*: EvalSuccess - - - InstantiateResultKind* {.pure.} = enum - `Error`, `Derivation` - `InstantiateResult`* {.preservesOr.} = object - case orKind*: InstantiateResultKind - of InstantiateResultKind.`Error`: - `error`*: Error + of EvalResultKind.`err`: + `err`*: Error - of InstantiateResultKind.`Derivation`: - `derivation`*: Derivation + of EvalResultKind.`ok`: + `ok`*: EvalSuccess ResolveStep* {.preservesRecord: "nix-actor".} = object `detail`*: ResolveDetail EvalFileResultKind* {.pure.} = enum - `Error`, `success` + `err`, `ok` `EvalFileResult`* {.preservesOr.} = object case orKind*: EvalFileResultKind - of EvalFileResultKind.`Error`: - `error`*: Error + of EvalFileResultKind.`err`: + `err`*: Error - of EvalFileResultKind.`success`: - `success`*: EvalFileSuccess + of EvalFileResultKind.`ok`: + `ok`*: EvalFileSuccess - Instantiate* {.preservesRecord: "instantiate".} = object - `expr`*: string - `log`* {.preservesEmbedded.}: Value - `result`*: InstantiateResult - - EvalFileSuccess* {.preservesTuple.} = object - `path`*: string - `args`*: Value + EvalFileSuccess* {.preservesRecord: "ok".} = object `result`*: Value - - Outputs* {.preservesRecord: "outputs".} = object - `drv`*: string - `storePaths`*: seq[string] + `args`*: Value + `path`*: string ResolveDetail* {.preservesDictionary.} = object `command-path`*: seq[string] @@ -97,26 +81,22 @@ type proc `$`*(x: Error | Eval | AttrSet | Realise | Derivation | RealiseResult | EvalSuccess | + RealiseSuccess | EvalFile | EvalResult | - InstantiateResult | ResolveStep | EvalFileResult | - Instantiate | EvalFileSuccess | - Outputs | ResolveDetail): string = `$`(toPreserves(x)) proc encode*(x: Error | Eval | AttrSet | Realise | Derivation | RealiseResult | EvalSuccess | + RealiseSuccess | EvalFile | EvalResult | - InstantiateResult | ResolveStep | EvalFileResult | - Instantiate | EvalFileSuccess | - Outputs | ResolveDetail): seq[byte] = encode(toPreserves(x)) diff --git a/test.pr b/test.pr index e4f6dfb6a6db..9e8589dad415 100755 --- a/test.pr +++ b/test.pr @@ -1,4 +1,14 @@ -#!/usr/bin/env -S syndicate-server --control --config +#!/usr/bin/env -S syndicate-server --config + +let ?nixConfig = { + command-path: [ "/run/current-system/sw/bin" ] + lookupPath: [ + "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos", + "nixos-config=/etc/nixos/configuration.nix", + "/nix/var/nix/profiles/per-user/root/channels", + ] + options: { } + } let ?nixLog = dataspace $nixLog ?? ?line [ @@ -10,37 +20,32 @@ $results ? ?any [ $log ! ] -? $cap [ +> +? >> [ - + $nix - - - {}; in pkgs.cowsay" $nixLog $results> + $nix # Realise all observed store derivations. - $results ? [ - $cap + $results ? > [ + $nix ] ] -let ?resolver = <* $config [ >]> -? ?cap> [ - $cap $resolver> +? > [ + $log ! + > + ? ?obj> [ + let ?rewriter = <* $config [>>]> + $obj $rewriter> + ] ] -> - +# The authors build system creates this file. > + ? [ Date: Sun, 30 Jun 2024 14:24:42 +0300 Subject: Improved translation of Preserves to Nix --- sbom.json | 2 +- src/nix_actor.nim | 20 ++++++++------- src/nix_actor/nix_api.nim | 49 ++++++++++++++++++++++++++++++------- src/nix_actor/nix_values.nim | 58 +++++++++++++++++++++++++++++++++++++++++++- test.pr | 4 +-- 5 files changed, 111 insertions(+), 22 deletions(-) diff --git a/sbom.json b/sbom.json index 478ef9fbdce0..591aa4bce14c 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240629", + "version": "20240630", "authors": [ { "name": "Emery Hemingway" diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 5c03e0c16071..40977949f56b 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -81,23 +81,25 @@ proc eval(store: Store; state: EvalState; expr: string): EvalResult = finally: close(nixVal) -proc evalFile(store: Store; state: EvalState; path: string; args: Value): EvalFileResult = +proc evalFile(store: Store; state: EvalState; path: string; prArgs: Value): EvalFileResult = var - nixVal: NixValue + fn, arg, res: NixValue try: - var expr = """import $1 (builtins.fromJSON ''$2'')""" % [ path, args.jsonText ] - # TODO: convert to NixValue instead of using JSON conversion. - nixVal = state.evalFromString(expr, "") - state.force(nixVal) + arg = prArgs.toNix(state) + fn = state.evalFromString("import " & path, "") + res = apply(state, fn, arg) + state.force(res) result = EvalFileResult(orKind: EvalFileResultKind.ok) - result.ok.result = nixVal.toPreserves(state).unthunkAll - result.ok.args = args + result.ok.result = res.toPreserves(state).unthunkAll + result.ok.args = prArgs result.ok.path = path except CatchableError as err: reset result result.err.message = err.msg finally: - close(nixVal) + close(res) + close(arg) + close(fn) proc serve(turn: Turn; detail: ResolveDetail; store: Store; state: EvalState; ds: Cap) = during(turn, ds, Eval.grabWithin) do (expr: string, resp: Cap): diff --git a/src/nix_actor/nix_api.nim b/src/nix_actor/nix_api.nim index d4700ef2da55..7abdbd5ce576 100644 --- a/src/nix_actor/nix_api.nim +++ b/src/nix_actor/nix_api.nim @@ -1,7 +1,12 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -import ./nix_api_types, ./nix_api_value, ./nix_api_store, ./nix_api_expr +import + ./nix_api_expr, + ./nix_api_store, + ./nix_api_types, + ./nix_api_util, + ./nix_api_value export NixContext, Store, EvalState, Value, ValueType, gc_decref, isNil @@ -9,7 +14,16 @@ export NixContext, Store, EvalState, Value, ValueType, {.passC: staticExec"$PKG_CONFIG --cflags nix-expr-c".} {.passL: staticExec"$PKG_CONFIG --libs nix-expr-c".} -# Always pass NixContext as a nil pointer. +proc check*(ctx: NixContext; err: nix_err) = + if err != NIX_OK: + assert not ctx.isNil + var + n: cuint + p = err_msg(NixContext(nil), ctx, addr n) + msg = newString(n) + if n > 0: + copyMem(msg[0].addr, p, msg.len) + raise newException(NixException, msg) proc initLibexpr* = var ctx: NixContext @@ -35,18 +49,26 @@ proc newState*(store: Store; lookupPath: openarray[string]): EvalState = proc close*(state: EvalState) = state_free(state) +proc close*(value: Value) = + var ctx: NixContext + discard gc_decref(ctx, cast[pointer](value)) + proc newValue*(state: EvalState): Value = var ctx: NixContext alloc_value(ctx, state) -proc evalFromString*(state: EvalState; expr, path: string): Value = - var ctx: NixContext - result = alloc_value(ctx, state) - discard expr_eval_from_string(ctx, state, expr, path, result) +proc evalFromString*(ctx: NixContext; state: EvalState; expr, path: string; result: Value) = + ctx.check expr_eval_from_string(ctx, state, expr, path, result) -proc close*(value: Value) = - var ctx: NixContext - discard gc_decref(ctx, cast[pointer](value)) +proc evalFromString*(state: EvalState; expr, path: string): Value = + let ctx = c_context_create() + try: + result = alloc_value(ctx, state) + evalFromString(ctx, state, expr, path, result) + except CatchableError as err: + c_context_free(ctx) + result.close() + raise err proc force*(state: EvalState; value: Value) = var ctx: NixContext @@ -55,3 +77,12 @@ proc force*(state: EvalState; value: Value) = proc get_attr_byidx*(ctx: NixContext; value: Value; state: EvalState; i: cuint): (cstring, Value) = var ctx: NixContext result[1] = get_attr_byidx(ctx, value, state, i, addr result[0]) + +proc apply*(ctx: NixContext; state: EvalState; fn, arg: Value): Value = + result = alloc_value(ctx, state) + ctx.check init_apply(ctx, result, fn, arg) + +proc apply*(state: EvalState; fn, arg: Value): Value = + let ctx = c_context_create() + defer: c_context_free(ctx) + apply(ctx, state, fn, arg) diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim index 5b426dd7d9ab..603bb6a620aa 100644 --- a/src/nix_actor/nix_values.nim +++ b/src/nix_actor/nix_values.nim @@ -34,7 +34,8 @@ proc unthunkAll*(v: Value): Value = v.mapEmbeds(unthunk) proc toPreserves*(value: NixValue; state: EvalState): Value {.gcsafe.} = - var ctx: NixContext + var ctx: NixContext # nil + # TODO: use a context for error handling let kind = get_type(ctx, value) case kind of NIX_TYPE_THUNK: raiseAssert "cannot preserve thunk" @@ -79,3 +80,58 @@ proc toPreserves*(value: NixValue; state: EvalState): Value {.gcsafe.} = result = "«function»".toPreserves of NIX_TYPE_EXTERNAL: result = "«external»".toPreserves + +proc translate*(ctx: NixContext; state: EvalState; pr: preserves.Value): NixValue = + try: + result = alloc_value(ctx, state) + case pr.kind + of pkBoolean: + ctx.check init_bool(ctx, result, pr.bool) + of pkFloat: + ctx.check init_float(ctx, result, pr.float.cdouble) + of pkRegister: + ctx.check init_int(ctx, result, pr.register.int64) + of pkBigInt: + ctx.check init_int(ctx, result, pr.register.int64) + of pkString: + ctx.check init_string(ctx, result, pr.string) + of pkByteString: + raise newException(ValueError, "cannot convert large Preserves integer to Nix: " & $pr) + of pkSymbol: + evalFromString(ctx, state, cast[string](pr.symbol), "", result) + of pkRecord: + if pr.isRecord("null", 0): + ctx.check init_null(ctx, result) + elif pr.isRecord("drv", 2): + let b = make_bindings_builder(ctx, state, 2) + defer: bindings_builder_free(b) + ctx.check bindings_builder_insert(ctx, b, "drvPath", translate(ctx, state, pr.fields[0])) + ctx.check bindings_builder_insert(ctx, b, "outPath", translate(ctx, state, pr.fields[1])) + ctx.check make_attrs(ctx, result, b) + else: + raise newException(ValueError, "cannot convert Preserves record to Nix: " & $pr) + of pkSequence, pkSet: + let b = make_list_builder(ctx, state, pr.len.csize_t) + defer: list_builder_free(b) + for i, e in pr: + ctx.check list_builder_insert(ctx, b, i.register.cuint, translate(ctx, state, e)) + ctx.check make_list(ctx, b, result) + of pkDictionary: + let b = make_bindings_builder(ctx, state, pr.dict.len.csize_t) + defer: bindings_builder_free(b) + for (name, value) in pr.dict: + if name.isSymbol: + ctx.check bindings_builder_insert(ctx, b, name.symbol.string, translate(ctx, state, value)) + else: + ctx.check bindings_builder_insert(ctx, b, $name, translate(ctx, state, value)) + ctx.check make_attrs(ctx, result, b) + of pkEmbedded: + raise newException(ValueError, "cannot convert Preserves embedded value to Nix") + except CatchableError as err: + result.close() + raise err + +proc toNix*(pr: preserves.Value; state: EvalState): NixValue = + let ctx = c_context_create() + defer: c_context_free(ctx) + result = translate(ctx, state, pr) diff --git a/test.pr b/test.pr index 9e8589dad415..ce8b0916ef14 100755 --- a/test.pr +++ b/test.pr @@ -3,7 +3,7 @@ let ?nixConfig = { command-path: [ "/run/current-system/sw/bin" ] lookupPath: [ - "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos", + "nixpkgs=/home/repo/nixpkgs/channel", "nixos-config=/etc/nixos/configuration.nix", "/nix/var/nix/profiles/per-user/root/channels", ] @@ -25,7 +25,7 @@ $results ? ?any [ $nix - $nix + $nix # Realise all observed store derivations. $results ? > [ -- cgit 1.4.1 From 33d2f0137150620c060331da70b4739ed195175e Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sun, 30 Jun 2024 19:02:15 +0300 Subject: Close instantiate process on facet stop --- sbom.json | 2 +- src/nix_actor.nim | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sbom.json b/sbom.json index 591aa4bce14c..a7a1c58df9dc 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240630", + "version": "20240703", "authors": [ { "name": "Emery Hemingway" diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 40977949f56b..15699f4b17ef 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -45,6 +45,9 @@ proc realise(facet: Facet; detail: ResolveDetail; drv: string; log: Option[Cap], env = detail.commandlineEnv(), options = {}, ) + facet.onStop do (turn: Turn): + if p.running: + p.kill() var errors = errorStream(p) line = "".toPreserves @@ -53,7 +56,7 @@ proc realise(facet: Facet; detail: ResolveDetail; drv: string; log: Option[Cap], if log.isSome: facet.run do (turn: Turn): message(turn, log.get, line) - elif not running(p): break + elif not p.running: break initDuration(milliseconds = 250).some.runOnce var storePaths = p.outputStream.readAll.strip.split doAssert storePaths != @[] -- cgit 1.4.1 From b914bed397b03f1f8c531251a58f654dc90d5f20 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Wed, 3 Jul 2024 14:06:06 +0300 Subject: Contextual error handling I don't know how this Nix C API error handling works. --- src/nix_actor.nim | 66 ++++++++++---------- src/nix_actor/nix_api.nim | 89 +++++++++++---------------- src/nix_actor/nix_api_value.nim | 31 +++++----- src/nix_actor/nix_values.nim | 133 ++++++++++++++++++++-------------------- src/nix_actor/utils.nim | 24 ++++++++ 5 files changed, 172 insertions(+), 171 deletions(-) create mode 100644 src/nix_actor/utils.nim diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 15699f4b17ef..9c300010bfff 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -7,7 +7,7 @@ import pkg/syndicate, pkg/syndicate/protocols/gatekeeper, pkg/syndicate/relays, - ./nix_actor/[nix_api, nix_values], + ./nix_actor/[nix_api, nix_api_expr, nix_values], ./nix_actor/protocol proc echo(args: varargs[string, `$`]) {.used.} = @@ -71,44 +71,42 @@ proc realise(facet: Facet; detail: ResolveDetail; drv: string; log: Option[Cap], close(p) proc eval(store: Store; state: EvalState; expr: string): EvalResult = - var nixVal: NixValue - try: - nixVal = state.evalFromString(expr, "") - state.force(nixVal) - result = EvalResult(orKind: EvalResultKind.ok) - result.ok.result = nixVal.toPreserves(state).unthunkAll - result.ok.expr = expr - except CatchableError as err: - reset result - result.err.message = err.msg - finally: - close(nixVal) + defer: close(state) # Single-use state. + var nixVal: NixValue + try: + nixVal = state.evalFromString(expr, "") + state.force(nixVal) + result = EvalResult(orKind: EvalResultKind.ok) + result.ok.result = nixVal.toPreserves(state).unthunkAll + result.ok.expr = expr + except CatchableError as err: + reset result + result.err.message = err.msg proc evalFile(store: Store; state: EvalState; path: string; prArgs: Value): EvalFileResult = - var - fn, arg, res: NixValue - try: - arg = prArgs.toNix(state) - fn = state.evalFromString("import " & path, "") - res = apply(state, fn, arg) - state.force(res) - result = EvalFileResult(orKind: EvalFileResultKind.ok) - result.ok.result = res.toPreserves(state).unthunkAll - result.ok.args = prArgs - result.ok.path = path - except CatchableError as err: - reset result - result.err.message = err.msg - finally: - close(res) - close(arg) - close(fn) + defer: close(state) # Single-use state. + var + fn, arg, res: NixValue + try: + arg = prArgs.toNix(state) + fn = state.evalFromString("import " & path, "") + res = apply(state, fn, arg) + state.force(res) + result = EvalFileResult(orKind: EvalFileResultKind.ok) + result.ok.result = res.toPreserves(state).unthunkAll + result.ok.args = prArgs + result.ok.path = path + except CatchableError as err: + reset result + result.err.message = err.msg -proc serve(turn: Turn; detail: ResolveDetail; store: Store; state: EvalState; ds: Cap) = +proc serve(turn: Turn; detail: ResolveDetail; store: Store; ds: Cap) = during(turn, ds, Eval.grabWithin) do (expr: string, resp: Cap): + let state = newState(store, detail.lookupPath) discard publish(turn, resp, eval(store, state, expr)) during(turn, ds, EvalFile.grabWithin) do (path: string, args: Value, resp: Cap): + let state = newState(store, detail.lookupPath) discard publish(turn, resp, evalFile(store, state, path, args)) during(turn, ds, Realise.grabWithin) do (drv: string, log: Value, resp: Cap): @@ -123,14 +121,12 @@ proc main() = during(turn, relay, pat) do (detail: ResolveDetail; observer: Cap): let store = openStore() - state = newState(store, detail.lookupPath) ds = turn.newDataspace() # TODO: attenuate this dataspace to only the assertions we observe. linkActor(turn, "nix-actor") do (turn: Turn): - serve(turn, detail, store, state, ds) + serve(turn, detail, store, ds) discard publish(turn, observer, ResolvedAccepted(responderSession: ds)) do: - close(state) close(store) main() diff --git a/src/nix_actor/nix_api.nim b/src/nix_actor/nix_api.nim index 7abdbd5ce576..fe29989f1c53 100644 --- a/src/nix_actor/nix_api.nim +++ b/src/nix_actor/nix_api.nim @@ -5,8 +5,8 @@ import ./nix_api_expr, ./nix_api_store, ./nix_api_types, - ./nix_api_util, - ./nix_api_value + ./nix_api_value, + ./utils export NixContext, Store, EvalState, Value, ValueType, gc_decref, isNil @@ -14,75 +14,58 @@ export NixContext, Store, EvalState, Value, ValueType, {.passC: staticExec"$PKG_CONFIG --cflags nix-expr-c".} {.passL: staticExec"$PKG_CONFIG --libs nix-expr-c".} -proc check*(ctx: NixContext; err: nix_err) = - if err != NIX_OK: - assert not ctx.isNil - var - n: cuint - p = err_msg(NixContext(nil), ctx, addr n) - msg = newString(n) - if n > 0: - copyMem(msg[0].addr, p, msg.len) - raise newException(NixException, msg) - -proc initLibexpr* = - var ctx: NixContext - discard libexpr_init(ctx) +proc initLibexpr*() = + mitNix: + discard nix.libexpr_init() proc openStore*(uri: string, params: varargs[string]): Store = - var ctx: NixContext - var args = allocCStringArray(params) - defer: deallocCStringArray(args) - result = store_open(ctx, uri, addr args) + mitNix: + var args = allocCStringArray(params) + defer: deallocCStringArray(args) + result = nix.store_open(uri, addr args) proc openStore*(): Store = - var ctx: NixContext - result = store_open(ctx, nil, nil) + mitNix: + result = nix.store_open(nil, nil) proc close*(store: Store) = store_free(store) proc newState*(store: Store; lookupPath: openarray[string]): EvalState = - var ctx: NixContext - var path = allocCStringArray(lookupPath) - defer: deallocCStringArray(path) - result = state_create(ctx, path, store) + mitNix: + var path = allocCStringArray(lookupPath) + defer: deallocCStringArray(path) + result = nix.state_create(path, store) proc close*(state: EvalState) = state_free(state) proc close*(value: Value) = - var ctx: NixContext - discard gc_decref(ctx, cast[pointer](value)) + mitNix: + discard nix.gc_decref(cast[pointer](value)) -proc newValue*(state: EvalState): Value = - var ctx: NixContext - alloc_value(ctx, state) - -proc evalFromString*(ctx: NixContext; state: EvalState; expr, path: string; result: Value) = - ctx.check expr_eval_from_string(ctx, state, expr, path, result) +proc evalFromString*(nix: NixContext; state: EvalState; expr, path: string; result: Value) = + discard nix.expr_eval_from_string(state, expr, path, result) proc evalFromString*(state: EvalState; expr, path: string): Value = - let ctx = c_context_create() - try: - result = alloc_value(ctx, state) - evalFromString(ctx, state, expr, path, result) - except CatchableError as err: - c_context_free(ctx) - result.close() - raise err + mitNix: + try: + result = nix.alloc_value(state) + nix.evalFromString(state, expr, path, result) + except CatchableError as err: + result.close() + raise err proc force*(state: EvalState; value: Value) = - var ctx: NixContext - discard value_force(ctx, state, value) + mitNix: + discard nix.value_force(state, value) -proc get_attr_byidx*(ctx: NixContext; value: Value; state: EvalState; i: cuint): (cstring, Value) = - var ctx: NixContext - result[1] = get_attr_byidx(ctx, value, state, i, addr result[0]) +proc get_attr_byidx*(value: Value; state: EvalState; i: cuint): (cstring, Value) = + mitNix: + result[1] = nix.get_attr_byidx(value, state, i, addr result[0]) -proc apply*(ctx: NixContext; state: EvalState; fn, arg: Value): Value = - result = alloc_value(ctx, state) - ctx.check init_apply(ctx, result, fn, arg) +proc apply(nix: NixContext; state: EvalState; fn, arg: Value): Value = + result = nix.alloc_value(state) + discard nix.init_apply(result, fn, arg) proc apply*(state: EvalState; fn, arg: Value): Value = - let ctx = c_context_create() - defer: c_context_free(ctx) - apply(ctx, state, fn, arg) + mitNix: + result = nix.apply(state, fn, arg) diff --git a/src/nix_actor/nix_api_value.nim b/src/nix_actor/nix_api_value.nim index bed607a0f8c0..1a601d0d6a15 100644 --- a/src/nix_actor/nix_api_value.nim +++ b/src/nix_actor/nix_api_value.nim @@ -19,7 +19,7 @@ proc get_typename*(context: NixContext; value: Value): cstring {.nix_api_value.} proc get_bool*(context: NixContext; value: Value): bool {.nix_api_value.} -proc get_string*(context: NixContext; value: Value; callback: GetStringCallback; user_data: pointer): nix_err {.nix_api_value.} +proc get_string*(context: NixContext; value: Value; callback: GetStringCallback; user_data: pointer): nix_err {.nix_api_value, discardable.} proc get_path_string*(context: NixContext; value: Value): cstring {.nix_api_value.} @@ -43,39 +43,39 @@ proc get_attr_byidx*(context: NixContext; value: Value; state: EvalState; i: cui # proc get_attr_name_byidx*(context: NixContext; value: Value; state: EvalState; i: cuint): cstring {.nix_api_value.} -proc init_bool*(context: NixContext; value: Value; b: bool): nix_err {.nix_api_value.} +proc init_bool*(context: NixContext; value: Value; b: bool): nix_err {.nix_api_value, discardable.} -proc init_string*(context: NixContext; value: Value; str: cstring): nix_err {.nix_api_value.} +proc init_string*(context: NixContext; value: Value; str: cstring): nix_err {.nix_api_value, discardable.} -proc init_path_string*(context: NixContext; s: EvalState; value: Value; str: cstring): nix_err {.nix_api_value.} +proc init_path_string*(context: NixContext; s: EvalState; value: Value; str: cstring): nix_err {.nix_api_value, discardable.} -proc init_float*(context: NixContext; value: Value; d: cdouble): nix_err {.nix_api_value.} +proc init_float*(context: NixContext; value: Value; d: cdouble): nix_err {.nix_api_value, discardable.} -proc init_int*(context: NixContext; value: Value; i: int64): nix_err {.nix_api_value.} +proc init_int*(context: NixContext; value: Value; i: int64): nix_err {.nix_api_value, discardable.} -proc init_null*(context: NixContext; value: Value): nix_err {.nix_api_value.} +proc init_null*(context: NixContext; value: Value): nix_err {.nix_api_value, discardable.} -proc init_apply*(context: NixContext; value: Value; fn: Value; arg: Value): nix_err {.nix_api_value.} +proc init_apply*(context: NixContext; value: Value; fn: Value; arg: Value): nix_err {.nix_api_value, discardable.} -proc init_external*(context: NixContext; value: Value; val: ExternalValue): nix_err {.nix_api_value.} +proc init_external*(context: NixContext; value: Value; val: ExternalValue): nix_err {.nix_api_value, discardable.} -proc make_list*(context: NixContext; list_builder: ListBuilder; value: Value): nix_err {.nix_api_value.} +proc make_list*(context: NixContext; list_builder: ListBuilder; value: Value): nix_err {.nix_api_value, discardable.} proc make_list_builder*(context: NixContext; state: EvalState; capacity: csize_t): ListBuilder {.nix_api_value.} -proc list_builder_insert*(context: NixContext; list_builder: ListBuilder; index: cuint; value: Value): nix_err {.nix_api_value.} +proc list_builder_insert*(context: NixContext; list_builder: ListBuilder; index: cuint; value: Value): nix_err {.nix_api_value, discardable.} proc list_builder_free*(list_builder: ListBuilder) {.nix_api_value.} -proc make_attrs*(context: NixContext; value: Value; b: BindingsBuilder): nix_err {.nix_api_value.} +proc make_attrs*(context: NixContext; value: Value; b: BindingsBuilder): nix_err {.nix_api_value, discardable.} -# proc init_primop*(context: NixContext; value: Value; op: PrimOp): nix_err {.nix_api_value.} +# proc init_primop*(context: NixContext; value: Value; op: PrimOp): nix_err {.nix_api_value, discardable.} -proc copy_value*(context: NixContext; value: Value; source: Value): nix_err {.nix_api_value.} +proc copy_value*(context: NixContext; value: Value; source: Value): nix_err {.nix_api_value, discardable.} proc make_bindings_builder*(context: NixContext; state: EvalState; capacity: csize_t): BindingsBuilder {.nix_api_value.} -proc bindings_builder_insert*(context: NixContext; builder: BindingsBuilder; name: cstring; value: Value): nix_err {.nix_api_value.} +proc bindings_builder_insert*(context: NixContext; builder: BindingsBuilder; name: cstring; value: Value): nix_err {.nix_api_value, discardable.} proc bindings_builder_free*(builder: BindingsBuilder) {.nix_api_value.} @@ -90,4 +90,3 @@ proc realised_string_get_store_path_count*(realised_string: RealisedString): csi proc realised_string_get_store_path*(realised_string: RealisedString; index: csize_t): StorePath {.nix_api_value.} proc realised_string_free*(realised_string: RealisedString) {.nix_api_value.} - diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim index 603bb6a620aa..53890aec24e8 100644 --- a/src/nix_actor/nix_values.nim +++ b/src/nix_actor/nix_values.nim @@ -4,7 +4,7 @@ import std/options, pkg/preserves, - ./[nix_api, nix_api_util, nix_api_value] + ./[nix_api, nix_api_util, nix_api_value, utils] proc echo(args: varargs[string, `$`]) {.used.} = stderr.writeLine(args) @@ -34,97 +34,97 @@ proc unthunkAll*(v: Value): Value = v.mapEmbeds(unthunk) proc toPreserves*(value: NixValue; state: EvalState): Value {.gcsafe.} = - var ctx: NixContext # nil - # TODO: use a context for error handling - let kind = get_type(ctx, value) - case kind - of NIX_TYPE_THUNK: raiseAssert "cannot preserve thunk" - of NIX_TYPE_INT: - result = getInt(ctx, value).toPreserves - of NIX_TYPE_FLOAT: - result = getFloat(ctx, value).toPreserves - of NIX_TYPE_BOOL: - result = getBool(ctx, value).toPreserves - of NIX_TYPE_STRING: - let thunk = StringThunkRef() - let err = getString(ctx, value, thunkString, thunk[].addr) - doAssert err == NIX_OK, $err - result = thunk.embed - of NIX_TYPE_PATH: - result = ($getPathString(ctx, value)).toPreserves - of NIX_TYPE_NULL: - result = initRecord("null") - of NIX_TYPE_ATTRS: - if has_attr_byname(ctx, value, state, "drvPath"): - result = initRecord("drv", - get_attr_byname(ctx, value, state, "drvPath").toPreserves(state), - get_attr_byname(ctx, value, state, "outPath").toPreserves(state), - ) - else: - let n = getAttrsSize(ctx, value) - result = initDictionary(int n) + mitNix: + # TODO: use a context for error handling + let kind = nix.get_type(value) + case kind + of NIX_TYPE_THUNK: raiseAssert "cannot preserve thunk" + of NIX_TYPE_INT: + result = nix.getInt(value).toPreserves + of NIX_TYPE_FLOAT: + result = nix.getFloat(value).toPreserves + of NIX_TYPE_BOOL: + result = nix.getBool(value).toPreserves + of NIX_TYPE_STRING: + let thunk = StringThunkRef() + let err = nix.getString(value, thunkString, thunk[].addr) + doAssert err == NIX_OK, $err + result = thunk.embed + of NIX_TYPE_PATH: + result = ($nix.getPathString(value)).toPreserves + of NIX_TYPE_NULL: + result = initRecord("null") + of NIX_TYPE_ATTRS: + if nix.has_attr_byname(value, state, "drvPath"): + result = initRecord("drv", + nix.get_attr_byname(value, state, "drvPath").toPreserves(state), + nix.get_attr_byname(value, state, "outPath").toPreserves(state), + ) + else: + let n = nix.getAttrsSize(value) + result = initDictionary(int n) + var i: cuint + while i < n: + let (key, val) = get_attr_byidx(value, state, i) + result[($key).toSymbol] = val.toPreserves(state) + inc(i) + of NIX_TYPE_LIST: + let n = nix.getListSize(value) + result = initSequence(n) var i: cuint while i < n: - let (key, val) = get_attr_byidx(ctx, value, state, i) - result[($key).toSymbol] = val.toPreserves(state) + var val = nix.getListByIdx(value, state, i) + result[i] = val.toPreserves(state) inc(i) - of NIX_TYPE_LIST: - let n = getListSize(ctx, value) - result = initSequence(n) - var i: cuint - while i < n: - var val = getListByIdx(ctx, value, state, i) - result[i] = val.toPreserves(state) - inc(i) - of NIX_TYPE_FUNCTION: - result = "«function»".toPreserves - of NIX_TYPE_EXTERNAL: - result = "«external»".toPreserves + of NIX_TYPE_FUNCTION: + result = "«function»".toPreserves + of NIX_TYPE_EXTERNAL: + result = "«external»".toPreserves -proc translate*(ctx: NixContext; state: EvalState; pr: preserves.Value): NixValue = +proc translate*(nix: NixContext; state: EvalState; pr: preserves.Value): NixValue = try: - result = alloc_value(ctx, state) + result = nix.alloc_value(state) case pr.kind of pkBoolean: - ctx.check init_bool(ctx, result, pr.bool) + nix.init_bool(result, pr.bool) of pkFloat: - ctx.check init_float(ctx, result, pr.float.cdouble) + nix.init_float(result, pr.float.cdouble) of pkRegister: - ctx.check init_int(ctx, result, pr.register.int64) + nix.init_int(result, pr.register.int64) of pkBigInt: - ctx.check init_int(ctx, result, pr.register.int64) + nix.init_int(result, pr.register.int64) of pkString: - ctx.check init_string(ctx, result, pr.string) + nix.init_string(result, pr.string) of pkByteString: raise newException(ValueError, "cannot convert large Preserves integer to Nix: " & $pr) of pkSymbol: - evalFromString(ctx, state, cast[string](pr.symbol), "", result) + nix.evalFromString(state, cast[string](pr.symbol), "", result) of pkRecord: if pr.isRecord("null", 0): - ctx.check init_null(ctx, result) + nix.init_null(result) elif pr.isRecord("drv", 2): - let b = make_bindings_builder(ctx, state, 2) + let b = nix.make_bindings_builder(state, 2) defer: bindings_builder_free(b) - ctx.check bindings_builder_insert(ctx, b, "drvPath", translate(ctx, state, pr.fields[0])) - ctx.check bindings_builder_insert(ctx, b, "outPath", translate(ctx, state, pr.fields[1])) - ctx.check make_attrs(ctx, result, b) + nix.bindings_builder_insert(b, "drvPath", nix.translate(state, pr.fields[0])) + nix.bindings_builder_insert(b, "outPath", nix.translate(state, pr.fields[1])) + nix.make_attrs(result, b) else: raise newException(ValueError, "cannot convert Preserves record to Nix: " & $pr) of pkSequence, pkSet: - let b = make_list_builder(ctx, state, pr.len.csize_t) + let b = nix.make_list_builder(state, pr.len.csize_t) defer: list_builder_free(b) for i, e in pr: - ctx.check list_builder_insert(ctx, b, i.register.cuint, translate(ctx, state, e)) - ctx.check make_list(ctx, b, result) + discard nix.list_builder_insert(b, i.register.cuint, nix.translate(state, e)) + nix.make_list(b, result) of pkDictionary: - let b = make_bindings_builder(ctx, state, pr.dict.len.csize_t) + let b = nix.make_bindings_builder(state, pr.dict.len.csize_t) defer: bindings_builder_free(b) for (name, value) in pr.dict: if name.isSymbol: - ctx.check bindings_builder_insert(ctx, b, name.symbol.string, translate(ctx, state, value)) + nix.bindings_builder_insert(b, name.symbol.string, nix.translate(state, value)) else: - ctx.check bindings_builder_insert(ctx, b, $name, translate(ctx, state, value)) - ctx.check make_attrs(ctx, result, b) + nix.bindings_builder_insert(b, $name, nix.translate(state, value)) + nix.make_attrs(result, b) of pkEmbedded: raise newException(ValueError, "cannot convert Preserves embedded value to Nix") except CatchableError as err: @@ -132,6 +132,5 @@ proc translate*(ctx: NixContext; state: EvalState; pr: preserves.Value): NixValu raise err proc toNix*(pr: preserves.Value; state: EvalState): NixValue = - let ctx = c_context_create() - defer: c_context_free(ctx) - result = translate(ctx, state, pr) + mitNix: + result = nix.translate(state, pr) diff --git a/src/nix_actor/utils.nim b/src/nix_actor/utils.nim new file mode 100644 index 000000000000..fd28ba1e7b6a --- /dev/null +++ b/src/nix_actor/utils.nim @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +import + ./nix_api_types, + ./nix_api_util + +proc newException(ctx: NixContext): ref NixException = + new result + var + n: cuint + p = err_msg(NixContext(nil), ctx, addr n) + result.msg.setLen(n) + if n > 0: + copyMem(result.msg[0].addr, p, result.msg.len) + +template mitNix*(body: untyped): untyped = + ## Mit nix machen. + block: + var nix {.inject.} = c_context_create() + defer: c_context_free(nix) + body + if err_code(nix) != NIX_OK: + let err = newException(nix) -- cgit 1.4.1 From 23f8f5ee38867b7030997d5f04aaec1ab9602257 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Thu, 4 Jul 2024 11:11:15 +0300 Subject: Open store by URI --- protocol.prs | 9 ++++++--- sbom.json | 2 +- src/nix_actor.nim | 2 +- src/nix_actor/nix_api.nim | 21 +++++++++++++-------- src/nix_actor/protocol.nim | 1 + test.pr | 1 + 6 files changed, 23 insertions(+), 13 deletions(-) diff --git a/protocol.prs b/protocol.prs index 903ba55002ec..b9312d0ee388 100644 --- a/protocol.prs +++ b/protocol.prs @@ -6,9 +6,6 @@ ResolveDetail = { # PATH to search for Nix utilities. command-path: [string ...] - # Command line options. - options: AttrSet - # List of strings corresponding to entries in NIX_PATH. # For example: # [ "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos" @@ -16,6 +13,12 @@ ResolveDetail = { # "/nix/var/nix/profiles/per-user/root/channels" # ] lookupPath: [string ...] + + # Command line options. + options: AttrSet + + # Store selector, use "auto" unless you know exactly what you need. + store-uri: string } . # Common error type. diff --git a/sbom.json b/sbom.json index a7a1c58df9dc..71e9adb06b74 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240703", + "version": "20240704", "authors": [ { "name": "Emery Hemingway" diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 9c300010bfff..697a796e832f 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -120,7 +120,7 @@ proc main() = let pat = Resolve?:{ 0: ResolveStep.grabWithin, 1: grab() } during(turn, relay, pat) do (detail: ResolveDetail; observer: Cap): let - store = openStore() + store = openStore(detail.`store-uri`) ds = turn.newDataspace() # TODO: attenuate this dataspace to only the assertions we observe. linkActor(turn, "nix-actor") do (turn: Turn): diff --git a/src/nix_actor/nix_api.nim b/src/nix_actor/nix_api.nim index fe29989f1c53..c91bf29bde7f 100644 --- a/src/nix_actor/nix_api.nim +++ b/src/nix_actor/nix_api.nim @@ -14,19 +14,23 @@ export NixContext, Store, EvalState, Value, ValueType, {.passC: staticExec"$PKG_CONFIG --cflags nix-expr-c".} {.passL: staticExec"$PKG_CONFIG --libs nix-expr-c".} -proc initLibexpr*() = +proc initLibstore*() = mitNix: - discard nix.libexpr_init() + discard nix.libstore_init() -proc openStore*(uri: string, params: varargs[string]): Store = +proc initLibexpr*() = mitNix: - var args = allocCStringArray(params) - defer: deallocCStringArray(args) - result = nix.store_open(uri, addr args) + discard nix.libexpr_init() -proc openStore*(): Store = +proc openStore*(uri = "auto", params: varargs[string]): Store = mitNix: - result = nix.store_open(nil, nil) + if params.len == 0: + result = nix.store_open(uri, nil) + else: + var args = allocCStringArray(params) + defer: deallocCStringArray(args) + result = nix.store_open(uri, addr args) + assert not result.isNil proc close*(store: Store) = store_free(store) @@ -35,6 +39,7 @@ proc newState*(store: Store; lookupPath: openarray[string]): EvalState = var path = allocCStringArray(lookupPath) defer: deallocCStringArray(path) result = nix.state_create(path, store) + assert not result.isNil proc close*(state: EvalState) = state_free(state) diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 5635440ad6d8..1188ca238b65 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -78,6 +78,7 @@ type `command-path`*: seq[string] `lookupPath`*: seq[string] `options`*: AttrSet + `store-uri`*: string proc `$`*(x: Error | Eval | AttrSet | Realise | Derivation | RealiseResult | EvalSuccess | diff --git a/test.pr b/test.pr index ce8b0916ef14..cfb665c043ba 100755 --- a/test.pr +++ b/test.pr @@ -8,6 +8,7 @@ let ?nixConfig = { "/nix/var/nix/profiles/per-user/root/channels", ] options: { } + store-uri: "auto" } let ?nixLog = dataspace -- cgit 1.4.1 From df5dcaf0836713da22e426732a4bb37a25d56ded Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Thu, 4 Jul 2024 15:16:53 +0300 Subject: Realise using store API --- protocol.prs | 4 +-- src/nix_actor.nim | 58 ++++++++++++++--------------------------- src/nix_actor/nix_api_store.nim | 20 +++++++++----- src/nix_actor/nix_api_types.nim | 9 ++++--- src/nix_actor/nix_api_value.nim | 1 - src/nix_actor/protocol.nim | 4 +-- test.pr | 5 ++-- 7 files changed, 46 insertions(+), 55 deletions(-) diff --git a/protocol.prs b/protocol.prs index b9312d0ee388..927146e0e3b9 100644 --- a/protocol.prs +++ b/protocol.prs @@ -48,8 +48,8 @@ Derivation = . # Asserted to nix-actor. # @drvPath is realised and each output is asserted to @outputs. -Realise = . +Realise = . RealiseResult = Error / RealiseSuccess . -RealiseSuccess = . +RealiseSuccess = . AttrSet = {symbol: any ...:...} . diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 697a796e832f..852ae39d4263 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -3,11 +3,11 @@ import std/[options, os, osproc, streams, strtabs, strutils, tables, times], - pkg/preserves, + pkg/preserves, pkg/preserves/sugar, pkg/syndicate, pkg/syndicate/protocols/gatekeeper, pkg/syndicate/relays, - ./nix_actor/[nix_api, nix_api_expr, nix_values], + ./nix_actor/[nix_api, nix_api_expr, nix_api_store, nix_values, utils], ./nix_actor/protocol proc echo(args: varargs[string, `$`]) {.used.} = @@ -35,40 +35,20 @@ proc commandlineArgs(detail: ResolveDetail; args: varargs[string]): seq[string] proc commandlineEnv(detail: ResolveDetail): StringTableRef = newStringTable({"NIX_PATH": detail.lookupPath.join ":"}) -proc realise(facet: Facet; detail: ResolveDetail; drv: string; log: Option[Cap], resp: Cap) = - # TODO: run this process asynchronously with nim-sys. - var p: Process - try: - p = startProcess( - detail.findCommand("nix-store"), - args = detail.commandlineArgs("--realise", drv), - env = detail.commandlineEnv(), - options = {}, - ) - facet.onStop do (turn: Turn): - if p.running: - p.kill() - var - errors = errorStream(p) - line = "".toPreserves - while true: - if errors.readLine(line.string): - if log.isSome: - facet.run do (turn: Turn): - message(turn, log.get, line) - elif not p.running: break - initDuration(milliseconds = 250).some.runOnce - var storePaths = p.outputStream.readAll.strip.split - doAssert storePaths != @[] - facet.run do (turn: Turn): - for path in storePaths: - discard publish(turn, resp, RealiseSuccess( - storePath: path, drvPath: drv)) - except CatchableError as err: - facet.run do (turn: Turn): - discard publish(turn, resp, Error(message: err.msg)) - finally: - close(p) +proc realiseCallback(userdata: pointer; a, b: cstring) {.cdecl.} = + echo "realiseCallback(nil, ", a, ", ", b, ")" + +proc realise(turn: Turn; store: Store; path: string; resp: Cap) = + mitNix: + let storePath = nix.store_parse_path(store, path) + assert not storePath.isNil + defer: store_path_free(storePath) + turn.facet.run do (turn: Turn): + discard nix.store_realise(store, storePath, nil, realiseCallback) + if nix.store_is_valid_path(store, storePath): + discard publish(turn, resp, initRecord("ok", %path)) + else: + discard publish(turn, resp, initRecord("error", %"not a valid store path")) proc eval(store: Store; state: EvalState; expr: string): EvalResult = defer: close(state) # Single-use state. @@ -109,10 +89,11 @@ proc serve(turn: Turn; detail: ResolveDetail; store: Store; ds: Cap) = let state = newState(store, detail.lookupPath) discard publish(turn, resp, evalFile(store, state, path, args)) - during(turn, ds, Realise.grabWithin) do (drv: string, log: Value, resp: Cap): - realise(turn.facet, detail, drv, log.unembed(Cap), resp) + during(turn, ds, Realise.grabWithin) do (drv: string, resp: Cap): + realise(turn, store, drv, resp) proc main() = + initLibstore() initLibexpr() runActor("main") do (turn: Turn): @@ -122,7 +103,6 @@ proc main() = let store = openStore(detail.`store-uri`) ds = turn.newDataspace() - # TODO: attenuate this dataspace to only the assertions we observe. linkActor(turn, "nix-actor") do (turn: Turn): serve(turn, detail, store, ds) discard publish(turn, observer, ResolvedAccepted(responderSession: ds)) diff --git a/src/nix_actor/nix_api_store.nim b/src/nix_actor/nix_api_store.nim index 904621041dfb..81e95fd89f04 100644 --- a/src/nix_actor/nix_api_store.nim +++ b/src/nix_actor/nix_api_store.nim @@ -4,23 +4,31 @@ import ./nix_api_types {.pragma: nix_api_store, header: "nix_api_store.h", importc: "nix_$1".} -proc libstore_init*(context: NixContext): nix_err {.nix_api_store.} +proc libstore_init*(context: NixContext): nix_err {.nix_api_store, discardable.} -proc init_plugins*(context: NixContext): nix_err {.nix_api_store.} +proc libstore_init_no_load_config*(context: NixContext): nix_err {.nix_api_store, discardable.} -proc store_open*(a1: NixContext; uri: cstring; params: ptr cstringArray): Store {.nix_api_store.} +proc init_plugins*(context: NixContext): nix_err {.nix_api_store, discardable.} + +proc store_open*(context: NixContext; uri: cstring; params: ptr cstringArray): Store {.nix_api_store.} proc store_free*(store: Store) {.nix_api_store.} -proc store_get_uri*(context: NixContext; store: Store; dest: cstring; n: cuint): nix_err {.nix_api_store.} +proc store_get_uri*(context: NixContext; store: Store; callback: GetStringCallback; user_data: pointer): nix_err {.nix_api_store, discardable.} proc store_parse_path*(context: NixContext; store: Store; path: cstring): StorePath {.nix_api_store.} +proc store_path_name*(store_path: StorePath; callback: GetStringCallback; user_data: pointer) {.nix_api_store.} + +proc store_path_clone*(p: StorePath): StorePath {.nix_api_store.} + proc store_path_free*(p: StorePath) {.nix_api_store.} proc store_is_valid_path*(context: NixContext; store: Store; path: StorePath): bool {.nix_api_store.} -proc store_build*(context: NixContext; store: Store; path: StorePath; userdata: pointer; callback: proc (userdata: pointer; outname: cstring; `out`: cstring)): nix_err {.nix_api_store.} +type RealiseCallback* = proc (userdata: pointer; outname: cstring; `out`: cstring) {.cdecl.} + +proc store_realise*(context: NixContext; store: Store; path: StorePath; userdata: pointer; callback: RealiseCallback): nix_err {.nix_api_store, discardable.} -proc store_get_version*(a1: NixContext; store: Store; dest: cstring; n: cuint): nix_err {.nix_api_store.} +proc store_get_version*(context: NixContext; store: Store; callback: GetStringCallback; user_data: pointer): nix_err {.nix_api_store, discardable.} diff --git a/src/nix_actor/nix_api_types.nim b/src/nix_actor/nix_api_types.nim index fcb25ba2ccfd..9623ba5e075b 100644 --- a/src/nix_actor/nix_api_types.nim +++ b/src/nix_actor/nix_api_types.nim @@ -1,8 +1,8 @@ type nix_err* = cint - NixException* = object of CatchableError - NixContext* {.header: "nix_api_util.h", importc: "nix_c_context".} = distinct pointer EvalState* {.header: "nix_api_expr.h", importc.} = distinct pointer + NixContext* {.header: "nix_api_util.h", importc: "nix_c_context".} = distinct pointer + NixException* = object of CatchableError Store* {.header: "nix_api_store.h", importc.} = distinct pointer StorePath* {.header: "nix_api_store.h", importc.} = distinct pointer Value* {.header: "nix_api_value.h", importc.} = distinct pointer @@ -19,7 +19,10 @@ type NIX_TYPE_FUNCTION, NIX_TYPE_EXTERNAL -proc isNil*(p: NixContext): bool {.borrow.} + GetStringCallback* = proc (start: cstring; n: cuint; data: pointer) {.cdecl.} + proc isNil*(p: EvalState): bool {.borrow.} +proc isNil*(p: NixContext): bool {.borrow.} proc isNil*(p: Store): bool {.borrow.} +proc isNil*(p: StorePath): bool {.borrow.} proc isNil*(p: Value): bool {.borrow.} diff --git a/src/nix_actor/nix_api_value.nim b/src/nix_actor/nix_api_value.nim index 1a601d0d6a15..5fba2af56a58 100644 --- a/src/nix_actor/nix_api_value.nim +++ b/src/nix_actor/nix_api_value.nim @@ -9,7 +9,6 @@ type ExternalValue* {.header: "nix_api_value.h", importc.} = distinct pointer ListBuilder* {.header: "nix_api_value.h", importc.} = distinct pointer RealisedString* {.header: "nix_api_value.h", importc: "nix_realised_string".} = distinct pointer - GetStringCallback* = proc (start: cstring; n: cuint; data: pointer) {.cdecl.} proc alloc_value*(context: NixContext; state: EvalState): Value {.nix_api_value.} diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 1188ca238b65..f0bbccb0e058 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -13,7 +13,6 @@ type AttrSet* = Table[Symbol, Value] Realise* {.preservesRecord: "realise".} = object `drvPath`*: string - `log`* {.preservesEmbedded.}: Value `outputs`* {.preservesEmbedded.}: Value Derivation* {.preservesRecord: "drv".} = object @@ -36,8 +35,9 @@ type `expr`*: string RealiseSuccess* {.preservesRecord: "ok".} = object - `storePath`*: string `drvPath`*: string + `outName`*: string + `outPath`*: string EvalFile* {.preservesRecord: "eval-file".} = object `path`*: string diff --git a/test.pr b/test.pr index cfb665c043ba..b31916848692 100755 --- a/test.pr +++ b/test.pr @@ -29,8 +29,9 @@ $results ? ?any [ $nix # Realise all observed store derivations. - $results ? > [ - $nix + $results ? > [ + # $nix + $nix ] ] -- cgit 1.4.1 From c78bd9e198db84b919c9796abb3b27458d58432f Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Thu, 4 Jul 2024 15:36:26 +0300 Subject: Add cfg output to default.nix --- default.nix | 10 +++++++++- service.pr.in | 13 +++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 service.pr.in diff --git a/default.nix b/default.nix index db1dec61e79c..9d278f027e40 100644 --- a/default.nix +++ b/default.nix @@ -7,8 +7,16 @@ let buildNimSbom = pkgs.callPackage ./build-nim-sbom.nix { }; in buildNimSbom (finalAttrs: { - name = "nix-actor"; + outputs = [ + "out" + "cfg" + ]; nativeBuildInputs = [ pkgs.pkg-config ]; buildInputs = [ pkgs.nixVersions.latest ]; src = if lib.inNixShell then null else lib.cleanSource ./.; + postInstall = '' + mkdir $cfg + export mainProgram="$out/bin/nix-actor" + substituteAll service.pr.in $cfg/service.pr + ''; }) ./sbom.json diff --git a/service.pr.in b/service.pr.in new file mode 100644 index 000000000000..b4a660d4ced9 --- /dev/null +++ b/service.pr.in @@ -0,0 +1,13 @@ + + +? > [ + > + ? ?obj> [ + let ?rewriter = <* $config [ $resp>>]> + $obj $rewriter> + ] +] -- cgit 1.4.1 From 02072ac6d22365aaa28b6114b180d4bf82c7abc3 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Wed, 10 Jul 2024 21:44:52 +0100 Subject: Replace evaluation with dataspace like repositories Remove evaluation and realisation assertions and replace with a gatekeeper that imports and expression from a file and exposes the contents using dataspace observation semantics. --- README.md | 13 ++++ protocol.prs | 41 ++++--------- sbom.json | 2 +- src/nix_actor.nim | 133 ++++++++++++++--------------------------- src/nix_actor/nix_api_expr.nim | 2 +- src/nix_actor/nix_values.nim | 75 ++++++++++++++++++----- src/nix_actor/protocol.nim | 114 ++++++++++------------------------- test.pr | 49 ++++++--------- 8 files changed, 182 insertions(+), 247 deletions(-) diff --git a/README.md b/README.md index e1166968e868..0ec4817be8b1 100644 --- a/README.md +++ b/README.md @@ -6,4 +6,17 @@ See [protocol.prs](./protocol.prs) for the Syndicate protocol [schema](https://p For an example configuration see [test.pr](./test.pr). +## Expressions as dataspaces + +The actor exposes on its initial capability a gatekeeper that resolves requests in the form ``. The resolved entity responds to observations as if it were a dataspace by asserting back lazily evaluated values from the imported expression. + +### Caveats +- Functions are not observable, unless the function can be satisfied with the empty attrset `{}` as an argument. +- An observation that stops at an attrset with an `outPath` yields the `outPath` and not the attrset. This prevents abritrary recursion into derivation dependencies. + +### TODO +Realise store-paths from expressions using local and remote buils. + +## Worker protocol + The was once an abstraction of the Nix worker socket that could intermediate between clients and the worker but that code has been removed, refer to git history for that. diff --git a/protocol.prs b/protocol.prs index 927146e0e3b9..f1e0e9442f08 100644 --- a/protocol.prs +++ b/protocol.prs @@ -1,10 +1,12 @@ version 1 . -# Gatekeeper step to access nix-actor. -ResolveStep = . -ResolveDetail = { - # PATH to search for Nix utilities. - command-path: [string ...] +# Gatekeeper step to access a Nix repository. +RepoResolveStep = . +RepoResolveDetail = { + + # Path to a repository to import. + # This string is evaluated so it can include "<…>" paths. + import: string # List of strings corresponding to entries in NIX_PATH. # For example: @@ -14,29 +16,16 @@ ResolveDetail = { # ] lookupPath: [string ...] - # Command line options. - options: AttrSet + # Store URI selector, use "auto" unless you know exactly what you need. + store: string +} & @args RepoArgs . - # Store selector, use "auto" unless you know exactly what you need. - store-uri: string -} . +# Arguments to call the imported expression with if it is a function. +RepoArgs = @present { args: any } / @absent {} . # Common error type. Error = . -# Asserted to nix-actor. -# @expr is evaluated and asserted to @result. -Eval = . -EvalResult = @err Error / @ok EvalSuccess . -EvalSuccess = . - -# Asserted to nix-actor. -# @file is imported as a function, @args are applied, and the result asserted -# to @result. -EvalFile = . -EvalFileResult = @err Error / @ok EvalFileSuccess . -EvalFileSuccess = . - # Represents a Nix derivation. # The @storePath can be realized as store object from @drvPath. # @@ -46,10 +35,4 @@ EvalFileSuccess = . # complete dependency trees. Derivation = . -# Asserted to nix-actor. -# @drvPath is realised and each output is asserted to @outputs. -Realise = . -RealiseResult = Error / RealiseSuccess . -RealiseSuccess = . - AttrSet = {symbol: any ...:...} . diff --git a/sbom.json b/sbom.json index 71e9adb06b74..0f24056e140b 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240704", + "version": "20240710", "authors": [ { "name": "Emery Hemingway" diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 852ae39d4263..0c76b762fd67 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -6,7 +6,7 @@ import pkg/preserves, pkg/preserves/sugar, pkg/syndicate, pkg/syndicate/protocols/gatekeeper, - pkg/syndicate/relays, + pkg/syndicate/[patterns, relays], ./nix_actor/[nix_api, nix_api_expr, nix_api_store, nix_values, utils], ./nix_actor/protocol @@ -15,82 +15,45 @@ proc echo(args: varargs[string, `$`]) {.used.} = type Value = preserves.Value - -proc findCommand(detail: ResolveDetail; cmd: string): string = - for dir in detail.`command-path`: - result = dir / cmd - if result.fileExists: - return - raise newException(OSError, "could not find " & cmd) - -proc commandlineArgs(detail: ResolveDetail; args: varargs[string]): seq[string] = - result = newSeqOfCap[string](detail.options.len * 2 + args.len) - for sym, val in detail.options: - result.add("--" & $sym) - if not val.isString "": - result.add(val.jsonText) - for arg in args: - result.add arg - -proc commandlineEnv(detail: ResolveDetail): StringTableRef = - newStringTable({"NIX_PATH": detail.lookupPath.join ":"}) - -proc realiseCallback(userdata: pointer; a, b: cstring) {.cdecl.} = - echo "realiseCallback(nil, ", a, ", ", b, ")" - -proc realise(turn: Turn; store: Store; path: string; resp: Cap) = - mitNix: - let storePath = nix.store_parse_path(store, path) - assert not storePath.isNil - defer: store_path_free(storePath) - turn.facet.run do (turn: Turn): - discard nix.store_realise(store, storePath, nil, realiseCallback) - if nix.store_is_valid_path(store, storePath): - discard publish(turn, resp, initRecord("ok", %path)) - else: - discard publish(turn, resp, initRecord("error", %"not a valid store path")) - -proc eval(store: Store; state: EvalState; expr: string): EvalResult = - defer: close(state) # Single-use state. - var nixVal: NixValue - try: - nixVal = state.evalFromString(expr, "") - state.force(nixVal) - result = EvalResult(orKind: EvalResultKind.ok) - result.ok.result = nixVal.toPreserves(state).unthunkAll - result.ok.expr = expr - except CatchableError as err: - reset result - result.err.message = err.msg - -proc evalFile(store: Store; state: EvalState; path: string; prArgs: Value): EvalFileResult = - defer: close(state) # Single-use state. - var - fn, arg, res: NixValue - try: - arg = prArgs.toNix(state) - fn = state.evalFromString("import " & path, "") - res = apply(state, fn, arg) - state.force(res) - result = EvalFileResult(orKind: EvalFileResultKind.ok) - result.ok.result = res.toPreserves(state).unthunkAll - result.ok.args = prArgs - result.ok.path = path - except CatchableError as err: - reset result - result.err.message = err.msg - -proc serve(turn: Turn; detail: ResolveDetail; store: Store; ds: Cap) = - during(turn, ds, Eval.grabWithin) do (expr: string, resp: Cap): - let state = newState(store, detail.lookupPath) - discard publish(turn, resp, eval(store, state, expr)) - - during(turn, ds, EvalFile.grabWithin) do (path: string, args: Value, resp: Cap): - let state = newState(store, detail.lookupPath) - discard publish(turn, resp, evalFile(store, state, path, args)) - - during(turn, ds, Realise.grabWithin) do (drv: string, resp: Cap): - realise(turn, store, drv, resp) + RepoEntity = ref object of Entity + self: Cap + store: Store + state: EvalState + root: NixValue + +proc newRepoEntity(turn: Turn; detail: RepoResolveDetail): RepoEntity = + let entity = RepoEntity() + turn.onStop do (turn: Turn): + if not entity.state.isNil: + entity.state.close() + if not entity.store.isNil: + entity.store.close() + entity.store = openStore(detail.store) + entity.state = newState(entity.store, detail.lookupPath) + entity.root = entity.state.evalFromString("import " & detail.`import`, "") + if detail.args.isSome: + var na = detail.args.get.toNix(entity.state) + entity.root = entity.state.apply(entity.root, na) + entity.self = newCap(turn, entity) + entity + +method publish(repo: RepoEntity; turn: Turn; a: AssertionRef; h: Handle) = + ## Respond to observations with dataspace semantics, minus retraction + ## of assertions in response to the retraction of observations. + ## This entity is scoped to immutable data so this shouldn't be a problem. + var obs: Observe + if obs.fromPreserves(a.value) and obs.observer of Cap: + var analysis = analyse(obs.pattern) + var captures = newSeq[Value](analysis.capturePaths.len) + for i, path in analysis.constPaths: + var v = repo.state.step(repo.root, path) + if v.isNone or v.get != analysis.constValues[i]: + return + for i, path in analysis.capturePaths: + var v = repo.state.step(repo.root, path) + if v.isSome: + captures[i] = v.get.unthunkAll + discard publish(turn, Cap obs.observer, captures) proc main() = initLibstore() @@ -98,15 +61,11 @@ proc main() = runActor("main") do (turn: Turn): resolveEnvironment(turn) do (turn: Turn; relay: Cap): - let pat = Resolve?:{ 0: ResolveStep.grabWithin, 1: grab() } - during(turn, relay, pat) do (detail: ResolveDetail; observer: Cap): - let - store = openStore(detail.`store-uri`) - ds = turn.newDataspace() - linkActor(turn, "nix-actor") do (turn: Turn): - serve(turn, detail, store, ds) - discard publish(turn, observer, ResolvedAccepted(responderSession: ds)) - do: - close(store) + + let resolvePat = Resolve?:{ 0: RepoResolveStep.grabWithin, 1: grab() } + during(turn, relay, resolvePat) do (detail: RepoResolveDetail; observer: Cap): + linkActor(turn, "nix-repo") do (turn: Turn): + let repo = newRepoEntity(turn, detail) + discard publish(turn, observer, ResolvedAccepted(responderSession: repo.self)) main() diff --git a/src/nix_actor/nix_api_expr.nim b/src/nix_actor/nix_api_expr.nim index 591a691ce8ed..8bb42c3d9349 100644 --- a/src/nix_actor/nix_api_expr.nim +++ b/src/nix_actor/nix_api_expr.nim @@ -22,7 +22,7 @@ proc state_free*(state: EvalState) {.nix_api_expr.} proc gc_incref*(context: NixContext; `object`: pointer): nix_err {.nix_api_expr.} -proc gc_decref*(context: NixContext; `object`: pointer): nix_err {.nix_api_expr.} +proc gc_decref*(context: NixContext; `object`: pointer|Value): nix_err {.nix_api_expr.} proc gc_now*() {.nix_api_expr.} diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim index 53890aec24e8..a9558521004a 100644 --- a/src/nix_actor/nix_values.nim +++ b/src/nix_actor/nix_values.nim @@ -6,9 +6,6 @@ import pkg/preserves, ./[nix_api, nix_api_util, nix_api_value, utils] -proc echo(args: varargs[string, `$`]) {.used.} = - stderr.writeLine(args) - type Value = preserves.Value NixValue* = nix_api.Value @@ -33,12 +30,30 @@ proc unthunk*(v: Value): Value = proc unthunkAll*(v: Value): Value = v.mapEmbeds(unthunk) -proc toPreserves*(value: NixValue; state: EvalState): Value {.gcsafe.} = +proc callThru(state: EvalState; nv: NixValue): NixValue = + result = nv + mitNix: + while true: + case nix.get_type(result) + of NIX_TYPE_THUNK: + state.force(result) + of NIX_TYPE_FUNCTION: + # Call functions with empty attrsets. + var + args = nix.alloc_value(state) + bb = nix.make_bindings_builder(state, 0) + discard nix.gc_decref(args) + doAssert nix.make_attrs(args, bb) == NIX_OK + bindings_builder_free(bb) + result = state.apply(result, args) + else: + return + +proc toPreserves*(state: EvalState; value: NixValue): Value {.gcsafe.} = + var value = callThru(state, value) mitNix: - # TODO: use a context for error handling let kind = nix.get_type(value) case kind - of NIX_TYPE_THUNK: raiseAssert "cannot preserve thunk" of NIX_TYPE_INT: result = nix.getInt(value).toPreserves of NIX_TYPE_FLOAT: @@ -55,18 +70,16 @@ proc toPreserves*(value: NixValue; state: EvalState): Value {.gcsafe.} = of NIX_TYPE_NULL: result = initRecord("null") of NIX_TYPE_ATTRS: - if nix.has_attr_byname(value, state, "drvPath"): - result = initRecord("drv", - nix.get_attr_byname(value, state, "drvPath").toPreserves(state), - nix.get_attr_byname(value, state, "outPath").toPreserves(state), - ) + if nix.has_attr_byname(value, state, "outPath"): + result = state.toPreserves(nix.get_attr_byname(value, state, "outPath")) + # TODO: eager string conversion with __toString attribute? else: let n = nix.getAttrsSize(value) result = initDictionary(int n) var i: cuint while i < n: let (key, val) = get_attr_byidx(value, state, i) - result[($key).toSymbol] = val.toPreserves(state) + result[($key).toSymbol] = state.toPreserves(val) inc(i) of NIX_TYPE_LIST: let n = nix.getListSize(value) @@ -74,10 +87,10 @@ proc toPreserves*(value: NixValue; state: EvalState): Value {.gcsafe.} = var i: cuint while i < n: var val = nix.getListByIdx(value, state, i) - result[i] = val.toPreserves(state) + result[i] = state.toPreserves(val) inc(i) - of NIX_TYPE_FUNCTION: - result = "«function»".toPreserves + of NIX_TYPE_THUNK, NIX_TYPE_FUNCTION: + raiseAssert "cannot preserve thunk or function" of NIX_TYPE_EXTERNAL: result = "«external»".toPreserves @@ -134,3 +147,35 @@ proc translate*(nix: NixContext; state: EvalState; pr: preserves.Value): NixValu proc toNix*(pr: preserves.Value; state: EvalState): NixValue = mitNix: result = nix.translate(state, pr) + +proc step*(state: EvalState; nv: NixValue; path: openarray[preserves.Value]): Option[preserves.Value] = + var nv = callThru(state, nv) + mitNix: + var + i = 0 + while i < path.len: + if nv.isNil: return + var kind = nix.get_type(nv) + case kind + of NIX_TYPE_ATTRS: + var key: string + case path[i].kind + of pkString: + key = path[i].string + of pkSymbol: + key = path[i].symbol.string + else: + key = $path[i] + if not nix.has_attr_byname(nv, state, key): return + var ctx: NixContext + nv = nix.get_attr_byname(nv, state, key) + inc i + of NIX_TYPE_LIST: + var ix: cuint + if not ix.fromPreserves(path[i]): return + nv = nix.get_list_byidx(nv, state, ix) + inc i + else: + raiseAssert("cannot step " & $kind) + return + result = state.toPreserves(nv).some diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index f0bbccb0e058..a8411fb5eecc 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -1,103 +1,49 @@ import - preserves, std/tables + preserves, std/tables, std/options type Error* {.preservesRecord: "error".} = object `message`*: string - Eval* {.preservesRecord: "eval".} = object - `expr`*: string - `result`* {.preservesEmbedded.}: Value - - AttrSet* = Table[Symbol, Value] - Realise* {.preservesRecord: "realise".} = object - `drvPath`*: string - `outputs`* {.preservesEmbedded.}: Value - - Derivation* {.preservesRecord: "drv".} = object - `drvPath`*: string - `storePath`*: string - - RealiseResultKind* {.pure.} = enum - `Error`, `RealiseSuccess` - `RealiseResult`* {.preservesOr.} = object - case orKind*: RealiseResultKind - of RealiseResultKind.`Error`: - `error`*: Error - - of RealiseResultKind.`RealiseSuccess`: - `realisesuccess`*: RealiseSuccess - - - EvalSuccess* {.preservesRecord: "ok".} = object - `result`*: Value - `expr`*: string - - RealiseSuccess* {.preservesRecord: "ok".} = object - `drvPath`*: string - `outName`*: string - `outPath`*: string - - EvalFile* {.preservesRecord: "eval-file".} = object - `path`*: string + RepoArgsKind* {.pure.} = enum + `present`, `absent` + RepoArgsPresent* {.preservesDictionary.} = object `args`*: Value - `result`* {.preservesEmbedded.}: Value - - EvalResultKind* {.pure.} = enum - `err`, `ok` - `EvalResult`* {.preservesOr.} = object - case orKind*: EvalResultKind - of EvalResultKind.`err`: - `err`*: Error - - of EvalResultKind.`ok`: - `ok`*: EvalSuccess + RepoArgsAbsent* {.preservesDictionary.} = object - ResolveStep* {.preservesRecord: "nix-actor".} = object - `detail`*: ResolveDetail - - EvalFileResultKind* {.pure.} = enum - `err`, `ok` - `EvalFileResult`* {.preservesOr.} = object - case orKind*: EvalFileResultKind - of EvalFileResultKind.`err`: - `err`*: Error + `RepoArgs`* {.preservesOr.} = object + case orKind*: RepoArgsKind + of RepoArgsKind.`present`: + `present`*: RepoArgsPresent - of EvalFileResultKind.`ok`: - `ok`*: EvalFileSuccess + of RepoArgsKind.`absent`: + `absent`*: RepoArgsAbsent - EvalFileSuccess* {.preservesRecord: "ok".} = object - `result`*: Value - `args`*: Value - `path`*: string + RepoResolveStep* {.preservesRecord: "nix-repo".} = object + `detail`*: RepoResolveDetail - ResolveDetail* {.preservesDictionary.} = object - `command-path`*: seq[string] + AttrSet* = Table[Symbol, Value] + RepoResolveDetailArgs* = Option[Value] + RepoResolveDetailImport* = string + RepoResolveDetailLookupPath* = seq[string] + RepoResolveDetailStore* = string + `RepoResolveDetail`* {.preservesDictionary.} = object + `args`*: Option[Value] + `import`*: string `lookupPath`*: seq[string] - `options`*: AttrSet - `store-uri`*: string + `store`*: string + + Derivation* {.preservesRecord: "drv".} = object + `drvPath`*: string + `storePath`*: string -proc `$`*(x: Error | Eval | AttrSet | Realise | Derivation | RealiseResult | - EvalSuccess | - RealiseSuccess | - EvalFile | - EvalResult | - ResolveStep | - EvalFileResult | - EvalFileSuccess | - ResolveDetail): string = +proc `$`*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoResolveDetail | + Derivation): string = `$`(toPreserves(x)) -proc encode*(x: Error | Eval | AttrSet | Realise | Derivation | RealiseResult | - EvalSuccess | - RealiseSuccess | - EvalFile | - EvalResult | - ResolveStep | - EvalFileResult | - EvalFileSuccess | - ResolveDetail): seq[byte] = +proc encode*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoResolveDetail | + Derivation): seq[byte] = encode(toPreserves(x)) diff --git a/test.pr b/test.pr index b31916848692..948cf03cbacc 100755 --- a/test.pr +++ b/test.pr @@ -1,47 +1,36 @@ #!/usr/bin/env -S syndicate-server --config -let ?nixConfig = { - command-path: [ "/run/current-system/sw/bin" ] +let ?nixStepDetail = { + import: "" lookupPath: [ "nixpkgs=/home/repo/nixpkgs/channel", "nixos-config=/etc/nixos/configuration.nix", "/nix/var/nix/profiles/per-user/root/channels", ] - options: { } - store-uri: "auto" + store: "auto" } -let ?nixLog = dataspace -$nixLog ?? ?line [ - $log ! -] - -let ?results = dataspace -$results ? ?any [ - $log ! -] - -> -? >> [ - - $nix - - $nix - - # Realise all observed store derivations. - $results ? > [ - # $nix - $nix +> +? > $repo [ + ? { hello: { meta: { homepage: ?x } } } [ + $log ! + ] + ? { nim: ?nim } [ + ? { nim: { passthru: ?passthru } } [ + ? { nim: { passthru: { nim: { buildInputs: [ ?bi ] } } } } [ + $log ! + ] + ] ] - ] -? > [ - $log ! +# Service instantiation. +? > [ + $log ! > ? ?obj> [ - let ?rewriter = <* $config [>>]> - $obj $rewriter> + let ?rewriter = <* $config [ $resp>>]> + $obj $rewriter> ] ] -- cgit 1.4.1 From 87771d243314658c77c39e0e71e8ac70ac9fdc44 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Wed, 14 Aug 2024 14:52:54 +0100 Subject: Drop direnv --- .envrc | 2 -- Tuprules.tup | 8 ++++--- sbom.json | 2 +- test.pr | 72 +++++++++++++++++++++++++++++++++++++----------------------- 4 files changed, 51 insertions(+), 33 deletions(-) delete mode 100644 .envrc diff --git a/.envrc b/.envrc deleted file mode 100644 index d324c24ca4a0..000000000000 --- a/.envrc +++ /dev/null @@ -1,2 +0,0 @@ -source_env .. -use nix diff --git a/Tuprules.tup b/Tuprules.tup index 1b889871461d..9df8af527b91 100644 --- a/Tuprules.tup +++ b/Tuprules.tup @@ -1,8 +1,10 @@ PROJECT_DIR = $(TUP_CWD) -NIM = $(DIRENV) $(NIM) - include ../syndicate-nim/depends.tup -NIM_FLAGS += --path:$(TUP_CWD)/../syndicate-nim/src + +PKG_CONFIG_PATH += @(PKG_CONFIG_PATH_nix) +NIM_CFG_LINES += "putenv:PKG_CONFIG=\"@(PKG_CONFIG)\"" +NIM_CFG_LINES += "putenv:PKG_CONFIG_PATH=\"$(PKG_CONFIG_PATH)\"" +NIM_CFG_LINES += "path:\"$(TUP_CWD)/../syndicate-nim/src\"" NIM_GROUPS += $(TUP_CWD)/ NIM_GROUPS += $(TUP_CWD)/ diff --git a/sbom.json b/sbom.json index 0f24056e140b..08d70fd3220f 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240710", + "version": "20240814", "authors": [ { "name": "Emery Hemingway" diff --git a/test.pr b/test.pr index 948cf03cbacc..ec20154ecfb8 100755 --- a/test.pr +++ b/test.pr @@ -4,46 +4,64 @@ let ?nixStepDetail = { import: "" lookupPath: [ "nixpkgs=/home/repo/nixpkgs/channel", - "nixos-config=/etc/nixos/configuration.nix", - "/nix/var/nix/profiles/per-user/root/channels", ] store: "auto" } -> -? > $repo [ - ? { hello: { meta: { homepage: ?x } } } [ +> + +? > [ + $log ! +] + +? > +$repo +[ + $log ! + ? { hello: { meta: { homepage: ?x } } } + [ $log ! ] - ? { nim: ?nim } [ - ? { nim: { passthru: ?passthru } } [ - ? { nim: { passthru: { nim: { buildInputs: [ ?bi ] } } } } [ + ? { nim: ?nim } + [ + ? { nim: { passthru: ?passthru } } + [ + ? { nim: { passthru: { nim: { buildInputs: [ ?bi ] } } } } + [ $log ! ] ] ] ] -# Service instantiation. -? > [ - $log ! +@"Service instantiation." +? > +[ > - ? ?obj> [ - let ?rewriter = <* $config [ $resp>>]> - $obj $rewriter> + ? ?obj> + [ + $log ! + $obj <* $config [ + >> + >> + ]> + ]>> ] ] -# The authors build system creates this file. -> - -? [ - -] +@"The authors build system creates this file." + > + ]> +}>> -- cgit 1.4.1 From 3788607b16b2de15ac4d2addee019f552cd7c4f9 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Wed, 14 Aug 2024 14:53:37 +0100 Subject: Initial nix-store service --- protocol.prs | 9 +++++ src/nix_actor.nim | 71 ++++++++++++++++++++++++++++++------ src/nix_actor/nix_api.nim | 13 ++++++- src/nix_actor/nix_values.nim | 87 ++++++++++++++++++++++++-------------------- src/nix_actor/protocol.nim | 21 ++++++++++- test.pr | 39 ++++++++++++++++++-- 6 files changed, 182 insertions(+), 58 deletions(-) diff --git a/protocol.prs b/protocol.prs index f1e0e9442f08..54a9bc01ead7 100644 --- a/protocol.prs +++ b/protocol.prs @@ -35,4 +35,13 @@ Error = . # complete dependency trees. Derivation = . +# Gatekeeper step to access a Nix store. +StoreResolveStep = . +StoreResolveDetail = { + params: AttrSet + uri: string +} . + +CheckStorePath = . + AttrSet = {symbol: any ...:...} . diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 0c76b762fd67..292185471f08 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -3,7 +3,8 @@ import std/[options, os, osproc, streams, strtabs, strutils, tables, times], - pkg/preserves, pkg/preserves/sugar, + pkg/preserves, + pkg/preserves/sugar, pkg/syndicate, pkg/syndicate/protocols/gatekeeper, pkg/syndicate/[patterns, relays], @@ -13,8 +14,9 @@ import proc echo(args: varargs[string, `$`]) {.used.} = stderr.writeLine(args) +type Value = preserves.Value + type - Value = preserves.Value RepoEntity = ref object of Entity self: Cap store: Store @@ -45,16 +47,55 @@ method publish(repo: RepoEntity; turn: Turn; a: AssertionRef; h: Handle) = if obs.fromPreserves(a.value) and obs.observer of Cap: var analysis = analyse(obs.pattern) var captures = newSeq[Value](analysis.capturePaths.len) - for i, path in analysis.constPaths: - var v = repo.state.step(repo.root, path) - if v.isNone or v.get != analysis.constValues[i]: - return - for i, path in analysis.capturePaths: - var v = repo.state.step(repo.root, path) - if v.isSome: - captures[i] = v.get.unthunkAll + block stepping: + for i, path in analysis.constPaths: + var v = repo.state.step(repo.root, path) + if v.isNone or v.get != analysis.constValues[i]: + let null = initRecord("null") + for v in captures.mitems: v = null + break stepping + for i, path in analysis.capturePaths: + var v = repo.state.step(repo.root, path) + if v.isSome: + captures[i] = v.get.unthunkAll + else: captures[i] = initRecord("null") discard publish(turn, Cap obs.observer, captures) +type + StoreEntity = ref object of Entity + self: Cap + store: Store + +proc openStore(uri: string; params: AttrSet): Store = + var + pairs = newSeq[string](params.len) + i: int + for (key, val) in params.pairs: + pairs[i] = $key & "=" & $val + inc i + openStore(uri, pairs) + +proc newStoreEntity(turn: Turn; detail: StoreResolveDetail): StoreEntity = + let entity = StoreEntity() + turn.onStop do (turn: Turn): + if not entity.store.isNil: + entity.store.close() + reset entity.store + entity.store = openStore(detail.uri, detail.params) + entity.self = newCap(turn, entity) + entity + +method publish(entity: StoreEntity; turn: Turn; a: AssertionRef; h: Handle) = + var + checkPath: CheckStorePath + continuation: Cap + if checkPath.fromPreserves(a.value) and continuation.fromPreserves(checkPath.valid): + try: + let v = entity.store.isValidPath(checkPath.path) + publish(turn, continuation, initRecord("ok", %v)) + except CatchableError as err: + publish(turn, continuation, initRecord("error", %err.msg)) + proc main() = initLibstore() initLibexpr() @@ -62,10 +103,16 @@ proc main() = runActor("main") do (turn: Turn): resolveEnvironment(turn) do (turn: Turn; relay: Cap): - let resolvePat = Resolve?:{ 0: RepoResolveStep.grabWithin, 1: grab() } - during(turn, relay, resolvePat) do (detail: RepoResolveDetail; observer: Cap): + let resolveRepoPat = Resolve?:{ 0: RepoResolveStep.grabWithin, 1: grab() } + during(turn, relay, resolveRepoPat) do (detail: RepoResolveDetail; observer: Cap): linkActor(turn, "nix-repo") do (turn: Turn): let repo = newRepoEntity(turn, detail) discard publish(turn, observer, ResolvedAccepted(responderSession: repo.self)) + let resolveStorePat = Resolve?:{ 0: StoreResolveStep.grabWithin, 1: grab() } + during(turn, relay, resolveStorePat) do (detail: StoreResolveDetail; observer: Cap): + linkActor(turn, "nix-store") do (turn: Turn): + let e = newStoreEntity(turn, detail) + discard publish(turn, observer, ResolvedAccepted(responderSession: e.self)) + main() diff --git a/src/nix_actor/nix_api.nim b/src/nix_actor/nix_api.nim index c91bf29bde7f..fdcf76075108 100644 --- a/src/nix_actor/nix_api.nim +++ b/src/nix_actor/nix_api.nim @@ -22,7 +22,7 @@ proc initLibexpr*() = mitNix: discard nix.libexpr_init() -proc openStore*(uri = "auto", params: varargs[string]): Store = +proc openStore*(uri = "auto", params: openarray[string] = []): Store = mitNix: if params.len == 0: result = nix.store_open(uri, nil) @@ -34,6 +34,17 @@ proc openStore*(uri = "auto", params: varargs[string]): Store = proc close*(store: Store) = store_free(store) +proc isValidPath*(store: Store; path: string): bool = + assert not store.isNil + assert path != "" + mitNix: + assert not nix.isNil + let sp = nix.store_parse_path(store, path) + if sp.isNil: + raise newException(CatchableError, "store_parse_path failed") + defer: store_path_free(sp) + result = nix.store_is_valid_path(store, sp) + proc newState*(store: Store; lookupPath: openarray[string]): EvalState = mitNix: var path = allocCStringArray(lookupPath) diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim index a9558521004a..9b2d6be18460 100644 --- a/src/nix_actor/nix_values.nim +++ b/src/nix_actor/nix_values.nim @@ -49,50 +49,57 @@ proc callThru(state: EvalState; nv: NixValue): NixValue = else: return -proc toPreserves*(state: EvalState; value: NixValue): Value {.gcsafe.} = +proc toPreserves*(state: EvalState; value: NixValue; nix: NixContext): Value {.gcsafe.} = var value = callThru(state, value) - mitNix: - let kind = nix.get_type(value) - case kind - of NIX_TYPE_INT: - result = nix.getInt(value).toPreserves - of NIX_TYPE_FLOAT: - result = nix.getFloat(value).toPreserves - of NIX_TYPE_BOOL: - result = nix.getBool(value).toPreserves - of NIX_TYPE_STRING: - let thunk = StringThunkRef() - let err = nix.getString(value, thunkString, thunk[].addr) - doAssert err == NIX_OK, $err - result = thunk.embed - of NIX_TYPE_PATH: - result = ($nix.getPathString(value)).toPreserves - of NIX_TYPE_NULL: - result = initRecord("null") - of NIX_TYPE_ATTRS: - if nix.has_attr_byname(value, state, "outPath"): - result = state.toPreserves(nix.get_attr_byname(value, state, "outPath")) - # TODO: eager string conversion with __toString attribute? - else: - let n = nix.getAttrsSize(value) - result = initDictionary(int n) - var i: cuint - while i < n: - let (key, val) = get_attr_byidx(value, state, i) - result[($key).toSymbol] = state.toPreserves(val) - inc(i) - of NIX_TYPE_LIST: - let n = nix.getListSize(value) - result = initSequence(n) + + let kind = nix.get_type(value) + case kind + of NIX_TYPE_INT: + result = nix.getInt(value).toPreserves + of NIX_TYPE_FLOAT: + result = nix.getFloat(value).toPreserves + of NIX_TYPE_BOOL: + result = nix.getBool(value).toPreserves + of NIX_TYPE_STRING: + let thunk = StringThunkRef() + let err = nix.getString(value, thunkString, thunk[].addr) + doAssert err == NIX_OK, $err + result = thunk.embed + of NIX_TYPE_PATH: + result = ($nix.getPathString(value)).toPreserves + of NIX_TYPE_NULL: + result = initRecord("null") + of NIX_TYPE_ATTRS: + if nix.has_attr_byname(value, state, "__toString"): + var str = nix.get_attr_byname(value, state, "__toString") + if nix.get_type(str) == NIX_TYPE_FUNCTION: + str = state.apply(str, value) + result = state.toPreserves(str, nix) + elif nix.has_attr_byname(value, state, "outPath"): + result = state.toPreserves(nix.get_attr_byname(value, state, "outPath"), nix) + else: + let n = nix.getAttrsSize(value) + result = initDictionary(int n) var i: cuint while i < n: - var val = nix.getListByIdx(value, state, i) - result[i] = state.toPreserves(val) + let (key, val) = get_attr_byidx(value, state, i) + result[($key).toSymbol] = state.toPreserves(val, nix) inc(i) - of NIX_TYPE_THUNK, NIX_TYPE_FUNCTION: - raiseAssert "cannot preserve thunk or function" - of NIX_TYPE_EXTERNAL: - result = "«external»".toPreserves + of NIX_TYPE_LIST: + let n = nix.getListSize(value) + result = initSequence(n) + var i: cuint + while i < n: + var val = nix.getListByIdx(value, state, i) + result[i] = state.toPreserves(val, nix) + inc(i) + of NIX_TYPE_THUNK, NIX_TYPE_FUNCTION: + raiseAssert "cannot preserve thunk or function" + of NIX_TYPE_EXTERNAL: + result = "«external»".toPreserves + +proc toPreserves*(state: EvalState; value: NixValue): Value {.gcsafe.} = + mitNix: result = toPreserves(state, value, nix) proc translate*(nix: NixContext; state: EvalState; pr: preserves.Value): NixValue = try: diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index a8411fb5eecc..59960cf969c5 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -40,10 +40,27 @@ type `drvPath`*: string `storePath`*: string + StoreResolveDetail* {.preservesDictionary.} = object + `params`*: AttrSet + `uri`*: string + + CheckStorePath* {.preservesRecord: "check-path".} = object + `path`*: string + `valid`* {.preservesEmbedded.}: Value + + StoreResolveStep* {.preservesRecord: "nix-store".} = object + `detail`*: StoreResolveDetail + proc `$`*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoResolveDetail | - Derivation): string = + Derivation | + StoreResolveDetail | + CheckStorePath | + StoreResolveStep): string = `$`(toPreserves(x)) proc encode*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoResolveDetail | - Derivation): seq[byte] = + Derivation | + StoreResolveDetail | + CheckStorePath | + StoreResolveStep): seq[byte] = encode(toPreserves(x)) diff --git a/test.pr b/test.pr index ec20154ecfb8..4b7183e6550d 100755 --- a/test.pr +++ b/test.pr @@ -1,4 +1,12 @@ -#!/usr/bin/env -S syndicate-server --config +#!/usr/bin/env -S syndicate-server --control --config + +let ?trace = dataspace +$trace +? ?any +[ + $log ! + ?- [ $log ! ] +] let ?nixStepDetail = { import: "" @@ -8,7 +16,7 @@ let ?nixStepDetail = { store: "auto" } -> +# > ? > [ $log ! @@ -34,7 +42,16 @@ $repo ] ] -@"Service instantiation." +let ?nixStoreDetail = { uri: "auto" params: {} } + +> +? > +$store +[ + >]> > +] + +@"Repo service instantiation." ? > [ > @@ -50,6 +67,22 @@ $repo ] ] +@"Store service instantiation." +? > +[ + > + ? ?obj> + [ + $log ! + $obj <* $config [ + >> + >> + ]> + ]>> + ] +] + @"The authors build system creates this file." Date: Wed, 14 Aug 2024 18:27:59 +0100 Subject: Observable uri and version at nix-store --- protocol.prs | 1 + src/nix_actor.nim | 33 ++++++++++++++++++++++++++------- src/nix_actor/nix_api.nim | 25 +++++++++++++++++++++++++ src/nix_actor/protocol.nim | 2 +- test.pr | 8 +++++++- 5 files changed, 60 insertions(+), 9 deletions(-) diff --git a/protocol.prs b/protocol.prs index 54a9bc01ead7..d8acbbabf147 100644 --- a/protocol.prs +++ b/protocol.prs @@ -1,4 +1,5 @@ version 1 . +embeddedType EntityRef.Cap . # Gatekeeper step to access a Nix repository. RepoResolveStep = . diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 292185471f08..62fe95270c75 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -85,16 +85,35 @@ proc newStoreEntity(turn: Turn; detail: StoreResolveDetail): StoreEntity = entity.self = newCap(turn, entity) entity +proc serve(entity: StoreEntity; turn: Turn; checkPath: CheckStorePath) = + try: + let v = entity.store.isValidPath(checkPath.path) + publish(turn, checkPath.valid.Cap, initRecord("ok", %v)) + except CatchableError as err: + publish(turn, checkPath.valid.Cap, initRecord("error", %err.msg)) + +proc serve(entity: StoreEntity; turn: Turn; obs: Observe) = + let facet = turn.facet + if obs.pattern.matches(initRecord("uri", %"")): + entity.store.getUri do (s: string): + facet.run do (turn: Turn): + publish(turn, obs.observer.Cap, obs.pattern.capture(initRecord("uri", %s)).get) + if obs.pattern.matches(initRecord("version", %"")): + entity.store.getVersion do (s: string): + facet.run do (turn: Turn): + publish(turn, obs.observer.Cap, obs.pattern.capture(initRecord("version", %s)).get) + method publish(entity: StoreEntity; turn: Turn; a: AssertionRef; h: Handle) = var + # orc doesn't handle this as a union object + observe: Observe checkPath: CheckStorePath - continuation: Cap - if checkPath.fromPreserves(a.value) and continuation.fromPreserves(checkPath.valid): - try: - let v = entity.store.isValidPath(checkPath.path) - publish(turn, continuation, initRecord("ok", %v)) - except CatchableError as err: - publish(turn, continuation, initRecord("error", %err.msg)) + if checkPath.fromPreserves(a.value): + entity.serve(turn, checkPath) + elif observe.fromPreserves(a.value): + entity.serve(turn, observe) + else: + echo "unhandled assertion ", a.value proc main() = initLibstore() diff --git a/src/nix_actor/nix_api.nim b/src/nix_actor/nix_api.nim index fdcf76075108..af6716329db4 100644 --- a/src/nix_actor/nix_api.nim +++ b/src/nix_actor/nix_api.nim @@ -14,6 +14,19 @@ export NixContext, Store, EvalState, Value, ValueType, {.passC: staticExec"$PKG_CONFIG --cflags nix-expr-c".} {.passL: staticExec"$PKG_CONFIG --libs nix-expr-c".} +type + StringCallback = proc (s: string) {.closure.} + StringCallbackState = object + callback: StringCallback + +proc receiveString(start: cstring; n: cuint; state: pointer) {.cdecl.} = + let state = cast[ptr StringCallbackState](state) + assert not state.isNil + var buf = newString(n) + if n > 0: + copyMem(buf[0].addr, start, buf.len) + state.callback(buf) + proc initLibstore*() = mitNix: discard nix.libstore_init() @@ -34,6 +47,18 @@ proc openStore*(uri = "auto", params: openarray[string] = []): Store = proc close*(store: Store) = store_free(store) +proc getUri*(store: Store; cb: StringCallback) = + mitNix: + let state = new StringCallbackState + state.callback = cb + discard nix.store_get_uri(store, receiveString, state[].addr) + +proc getVersion*(store: Store; cb: StringCallback) = + mitNix: + let state = new StringCallbackState + state.callback = cb + discard nix.store_get_version(store, receiveString, state[].addr) + proc isValidPath*(store: Store; path: string): bool = assert not store.isNil assert path != "" diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 59960cf969c5..85563562d8b5 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -46,7 +46,7 @@ type CheckStorePath* {.preservesRecord: "check-path".} = object `path`*: string - `valid`* {.preservesEmbedded.}: Value + `valid`* {.preservesEmbedded.}: EmbeddedRef StoreResolveStep* {.preservesRecord: "nix-store".} = object `detail`*: StoreResolveDetail diff --git a/test.pr b/test.pr index 4b7183e6550d..15150ba2f60e 100755 --- a/test.pr +++ b/test.pr @@ -48,7 +48,13 @@ let ?nixStoreDetail = { uri: "auto" params: {} } ? > $store [ - >]> > + ? ?any + [ + $log ! + ] + > + ]>> ] @"Repo service instantiation." -- cgit 1.4.1 From 4f8bb408972283c4cd0ffed734ed370af12aa036 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Thu, 15 Aug 2024 11:21:36 +0100 Subject: Add tests --- sbom.json | 2 +- tests/Tupfile | 2 ++ tests/test.nim | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 tests/Tupfile create mode 100644 tests/test.nim diff --git a/sbom.json b/sbom.json index 08d70fd3220f..415a17fd59d1 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240814", + "version": "20240815", "authors": [ { "name": "Emery Hemingway" diff --git a/tests/Tupfile b/tests/Tupfile new file mode 100644 index 000000000000..49d1bc1be290 --- /dev/null +++ b/tests/Tupfile @@ -0,0 +1,2 @@ +include_rules +: t*.nim |> !balls |> diff --git a/tests/test.nim b/tests/test.nim new file mode 100644 index 000000000000..aa75842b50a1 --- /dev/null +++ b/tests/test.nim @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +import + std/[unittest], + pkg/balls, + pkg/preserves, + ../src/nix_actor/[nix_api, nix_values, protocol] + +suite "libexpr": + initLibexpr() + + let + store = openStore() + state = newState(store, []) + + proc checkConversion(s: string) = + var nixVal = state.evalFromString(s, "") + state.force(nixVal) + nixVal.close() + var pr = state.toPreserves(nixVal) + pr = pr.unthunkAll + echo pr + + test "lists": + let samples = [ + "[]", + "[null]", + "[[]]", + "[ null [ null [null null null null null null null null ] null ] null ]", + ] + for s in samples: + test s: + checkConversion(s) + + test "attrsets": + let samples = [ + "{}", + "{a = {}; }", + "{a = { x = {}; }; b = null; c = null; d = null; e = null; }", + ] + for s in samples: + test s: + checkConversion(s) + + test "derivation": + let samples = [ + "let pkgs = import { }; in pkgs.hello" + ] + for s in samples: + test s: + checkConversion(s) + + test "large": + let samples = + "builtins.listToAttrs (builtins.genList (x: { name = toString x; value = null; }) 99)" + checkConversion(samples) + +type AddToStoreClientAttrs {.preservesDictionary.} = object + ## A subset of AddToStoreAttrs + `ca-method`: Symbol + eris: seq[byte] + name: string + +test "fromPreserve": + const raw = "{ca: > ca-method: |fixed:r:sha256| deriver: > eris: #[CgA1VVrR0k5gjgU1wKQKVZr1RkANf4zUva3vyc2wmLzhzuL8XqeUL0HE4W3aRpXNwXyFbaLxtXJiLCUWSyLjej+h] name: \"default-builder.sh\" narHash: > narSize: > references: [] registrationTime: > sigs: > ultimate: >}" + + let pr = parsePreserves(raw) + var attrs: AddToStoreClientAttrs + check fromPreserve(attrs, pr) + +suite "gatekeeper": + + test "nix-repo": + var step: RepoResolveStep + + check not step.fromPreserves(parsePreserves""" + ", lookupPath: ["nixpkgs=/home/repo/nixpkgs/channel"] }> + """) + + check not step.fromPreserves(parsePreserves""" + + """) + + check step.fromPreserves parsePreserves""" + ", lookupPath: ["nixpkgs=/home/repo/nixpkgs/channel"] }> + """ + + check step.fromPreserves parsePreserves""" + ", lookupPath: ["nixpkgs=/home/repo/nixpkgs/channel"] }> + """ + + check step.fromPreserves parsePreserves""" + ", lookupPath: ["nixpkgs=/home/repo/nixpkgs/channel"], store: "local" }> + """ -- cgit 1.4.1 From a427117ba0614877945555824b04afa257686bb9 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 19 Aug 2024 17:07:48 +0300 Subject: Optional args at nix-repo step --- protocol.prs | 9 ++-- sbom.json | 2 +- src/nix_actor.nim | 106 ++++++++++++++++++++++++--------------------- src/nix_actor/protocol.nim | 32 ++++++++++++-- 4 files changed, 91 insertions(+), 58 deletions(-) diff --git a/protocol.prs b/protocol.prs index d8acbbabf147..f660beae1ac4 100644 --- a/protocol.prs +++ b/protocol.prs @@ -17,12 +17,13 @@ RepoResolveDetail = { # ] lookupPath: [string ...] - # Store URI selector, use "auto" unless you know exactly what you need. - store: string -} & @args RepoArgs . +} & @args RepoArgs & @store RepoStore . # Arguments to call the imported expression with if it is a function. -RepoArgs = @present { args: any } / @absent {} . +RepoArgs = @present { args: any } / @absent { } . + +# Store uri or capability. +RepoStore = @uri { store: string } / @cap { store: #:any } / @absent { } . # Common error type. Error = . diff --git a/sbom.json b/sbom.json index 415a17fd59d1..b3542913c928 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240815", + "version": "20240819", "authors": [ { "name": "Emery Hemingway" diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 62fe95270c75..c67eb885b1fb 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -2,13 +2,12 @@ # SPDX-License-Identifier: Unlicense import - std/[options, os, osproc, streams, strtabs, strutils, tables, times], + std/[options, strutils, tables, times], pkg/preserves, pkg/preserves/sugar, pkg/syndicate, - pkg/syndicate/protocols/gatekeeper, - pkg/syndicate/[patterns, relays], - ./nix_actor/[nix_api, nix_api_expr, nix_api_store, nix_values, utils], + pkg/syndicate/[gatekeepers, patterns, relays], + ./nix_actor/[nix_api, nix_api_store, nix_values], ./nix_actor/protocol proc echo(args: varargs[string, `$`]) {.used.} = @@ -16,51 +15,6 @@ proc echo(args: varargs[string, `$`]) {.used.} = type Value = preserves.Value -type - RepoEntity = ref object of Entity - self: Cap - store: Store - state: EvalState - root: NixValue - -proc newRepoEntity(turn: Turn; detail: RepoResolveDetail): RepoEntity = - let entity = RepoEntity() - turn.onStop do (turn: Turn): - if not entity.state.isNil: - entity.state.close() - if not entity.store.isNil: - entity.store.close() - entity.store = openStore(detail.store) - entity.state = newState(entity.store, detail.lookupPath) - entity.root = entity.state.evalFromString("import " & detail.`import`, "") - if detail.args.isSome: - var na = detail.args.get.toNix(entity.state) - entity.root = entity.state.apply(entity.root, na) - entity.self = newCap(turn, entity) - entity - -method publish(repo: RepoEntity; turn: Turn; a: AssertionRef; h: Handle) = - ## Respond to observations with dataspace semantics, minus retraction - ## of assertions in response to the retraction of observations. - ## This entity is scoped to immutable data so this shouldn't be a problem. - var obs: Observe - if obs.fromPreserves(a.value) and obs.observer of Cap: - var analysis = analyse(obs.pattern) - var captures = newSeq[Value](analysis.capturePaths.len) - block stepping: - for i, path in analysis.constPaths: - var v = repo.state.step(repo.root, path) - if v.isNone or v.get != analysis.constValues[i]: - let null = initRecord("null") - for v in captures.mitems: v = null - break stepping - for i, path in analysis.capturePaths: - var v = repo.state.step(repo.root, path) - if v.isSome: - captures[i] = v.get.unthunkAll - else: captures[i] = initRecord("null") - discard publish(turn, Cap obs.observer, captures) - type StoreEntity = ref object of Entity self: Cap @@ -115,6 +69,60 @@ method publish(entity: StoreEntity; turn: Turn; a: AssertionRef; h: Handle) = else: echo "unhandled assertion ", a.value +type + RepoEntity = ref object of Entity + self: Cap + store: Store + state: EvalState + root: NixValue + +proc newRepoEntity(turn: Turn; detail: RepoResolveDetail): RepoEntity = + let entity = RepoEntity() + turn.onStop do (turn: Turn): + if not entity.state.isNil: + entity.state.close() + if not entity.store.isNil: + entity.store.close() + if detail.store.isSome: + var other = detail.store.get.unembed(StoreEntity) + if other.isSome: + entity.store = other.get.store + elif detail.store.get.isString: + entity.store = openStore(detail.store.get.string) + else: + raise newException(CatchableError, "invalid store parameter") + else: + entity.store = openStore() + entity.state = newState(entity.store, detail.lookupPath) + entity.root = entity.state.evalFromString("import " & detail.`import`, "") + if detail.args.isSome: + var na = detail.args.get.toNix(entity.state) + entity.root = entity.state.apply(entity.root, na) + entity.self = newCap(turn, entity) + entity + +method publish(repo: RepoEntity; turn: Turn; a: AssertionRef; h: Handle) = + ## Respond to observations with dataspace semantics, minus retraction + ## of assertions in response to the retraction of observations. + ## This entity is scoped to immutable data so this shouldn't be a problem. + var obs: Observe + if obs.fromPreserves(a.value) and obs.observer of Cap: + var analysis = analyse(obs.pattern) + var captures = newSeq[Value](analysis.capturePaths.len) + block stepping: + for i, path in analysis.constPaths: + var v = repo.state.step(repo.root, path) + if v.isNone or v.get != analysis.constValues[i]: + let null = initRecord("null") + for v in captures.mitems: v = null + break stepping + for i, path in analysis.capturePaths: + var v = repo.state.step(repo.root, path) + if v.isSome: + captures[i] = v.get.unthunkAll + else: captures[i] = initRecord("null") + discard publish(turn, Cap obs.observer, captures) + proc main() = initLibstore() initLibexpr() diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 85563562d8b5..bcc6f058988d 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -26,15 +26,37 @@ type `detail`*: RepoResolveDetail AttrSet* = Table[Symbol, Value] + RepoStoreKind* {.pure.} = enum + `uri`, `cap`, `absent` + RepoStoreUri* {.preservesDictionary.} = object + `store`*: string + + RepoStoreCap* {.preservesDictionary.} = object + `store`* {.preservesEmbedded.}: EmbeddedRef + + RepoStoreAbsent* {.preservesDictionary.} = object + + `RepoStore`* {.preservesOr.} = object + case orKind*: RepoStoreKind + of RepoStoreKind.`uri`: + `uri`*: RepoStoreUri + + of RepoStoreKind.`cap`: + `cap`* {.preservesEmbedded.}: RepoStoreCap + + of RepoStoreKind.`absent`: + `absent`*: RepoStoreAbsent + + RepoResolveDetailArgs* = Option[Value] RepoResolveDetailImport* = string RepoResolveDetailLookupPath* = seq[string] - RepoResolveDetailStore* = string + RepoResolveDetailStore* = Option[Value] `RepoResolveDetail`* {.preservesDictionary.} = object `args`*: Option[Value] `import`*: string `lookupPath`*: seq[string] - `store`*: string + `store`*: Option[Value] Derivation* {.preservesRecord: "drv".} = object `drvPath`*: string @@ -51,14 +73,16 @@ type StoreResolveStep* {.preservesRecord: "nix-store".} = object `detail`*: StoreResolveDetail -proc `$`*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoResolveDetail | +proc `$`*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoStore | + RepoResolveDetail | Derivation | StoreResolveDetail | CheckStorePath | StoreResolveStep): string = `$`(toPreserves(x)) -proc encode*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoResolveDetail | +proc encode*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoStore | + RepoResolveDetail | Derivation | StoreResolveDetail | CheckStorePath | -- cgit 1.4.1 From 203576104cfed7f4ad3696152a323cb9513f4810 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 19 Aug 2024 17:09:35 +0300 Subject: Use gatekeeper utility from syndicate library --- src/nix_actor.nim | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/nix_actor.nim b/src/nix_actor.nim index c67eb885b1fb..a867ce93ec78 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -129,17 +129,12 @@ proc main() = runActor("main") do (turn: Turn): resolveEnvironment(turn) do (turn: Turn; relay: Cap): + let gk = spawnGatekeeper(turn, relay) - let resolveRepoPat = Resolve?:{ 0: RepoResolveStep.grabWithin, 1: grab() } - during(turn, relay, resolveRepoPat) do (detail: RepoResolveDetail; observer: Cap): - linkActor(turn, "nix-repo") do (turn: Turn): - let repo = newRepoEntity(turn, detail) - discard publish(turn, observer, ResolvedAccepted(responderSession: repo.self)) - - let resolveStorePat = Resolve?:{ 0: StoreResolveStep.grabWithin, 1: grab() } - during(turn, relay, resolveStorePat) do (detail: StoreResolveDetail; observer: Cap): - linkActor(turn, "nix-store") do (turn: Turn): - let e = newStoreEntity(turn, detail) - discard publish(turn, observer, ResolvedAccepted(responderSession: e.self)) + gk.serve do (turn: Turn; step: StoreResolveStep) -> Resolved: + newStoreEntity(turn, step.detail).self.resolveAccepted + + gk.serve do (turn: Turn; step: RepoResolveStep) -> Resolved: + newRepoEntity(turn, step.detail).self.resolveAccepted main() -- cgit 1.4.1 From da0d91f727282e01570d7e34c442995eba52d704 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 19 Aug 2024 17:10:37 +0300 Subject: Add Result type --- protocol.prs | 5 ++++- src/nix_actor.nim | 2 +- src/nix_actor/protocol.nim | 18 +++++++++++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/protocol.prs b/protocol.prs index f660beae1ac4..c87d810bd532 100644 --- a/protocol.prs +++ b/protocol.prs @@ -26,7 +26,7 @@ RepoArgs = @present { args: any } / @absent { } . RepoStore = @uri { store: string } / @cap { store: #:any } / @absent { } . # Common error type. -Error = . +Error = . # Represents a Nix derivation. # The @storePath can be realized as store object from @drvPath. @@ -47,3 +47,6 @@ StoreResolveDetail = { CheckStorePath = . AttrSet = {symbol: any ...:...} . + +# Value. +Result = Error / . diff --git a/src/nix_actor.nim b/src/nix_actor.nim index a867ce93ec78..0cae4f69ecdf 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -44,7 +44,7 @@ proc serve(entity: StoreEntity; turn: Turn; checkPath: CheckStorePath) = let v = entity.store.isValidPath(checkPath.path) publish(turn, checkPath.valid.Cap, initRecord("ok", %v)) except CatchableError as err: - publish(turn, checkPath.valid.Cap, initRecord("error", %err.msg)) + publish(turn, checkPath.valid.Cap, Error(message: %err.msg)) proc serve(entity: StoreEntity; turn: Turn; obs: Observe) = let facet = turn.facet diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index bcc6f058988d..bcd76763d004 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -4,7 +4,7 @@ import type Error* {.preservesRecord: "error".} = object - `message`*: string + `message`*: Value RepoArgsKind* {.pure.} = enum `present`, `absent` @@ -66,6 +66,20 @@ type `params`*: AttrSet `uri`*: string + ResultKind* {.pure.} = enum + `Error`, `ok` + ResultOk* {.preservesRecord: "ok".} = object + `value`*: Value + + `Result`* {.preservesOr.} = object + case orKind*: ResultKind + of ResultKind.`Error`: + `error`*: Error + + of ResultKind.`ok`: + `ok`*: ResultOk + + CheckStorePath* {.preservesRecord: "check-path".} = object `path`*: string `valid`* {.preservesEmbedded.}: EmbeddedRef @@ -77,6 +91,7 @@ proc `$`*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoStore | RepoResolveDetail | Derivation | StoreResolveDetail | + Result | CheckStorePath | StoreResolveStep): string = `$`(toPreserves(x)) @@ -85,6 +100,7 @@ proc encode*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoStore | RepoResolveDetail | Derivation | StoreResolveDetail | + Result | CheckStorePath | StoreResolveStep): seq[byte] = encode(toPreserves(x)) -- cgit 1.4.1 From 0e2cd9afea883e587fa8ae82cf436c527fe52a4c Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 19 Aug 2024 17:10:59 +0300 Subject: WiP! Add copy-closure --- protocol.prs | 3 +++ src/nix_actor.nim | 16 ++++++++++++++++ src/nix_actor/nix_api.nim | 9 +++++++++ src/nix_actor/nix_api_store.nim | 1 + src/nix_actor/protocol.nim | 7 +++++++ 5 files changed, 36 insertions(+) diff --git a/protocol.prs b/protocol.prs index c87d810bd532..19c3dc2ef7a5 100644 --- a/protocol.prs +++ b/protocol.prs @@ -46,6 +46,9 @@ StoreResolveDetail = { CheckStorePath = . +# Assertion. The store that this asserted to will copy the closure of @storePath to @destination. When the copy completes or fails a Result value is asserted to @result. +CopyClosure = . + AttrSet = {symbol: any ...:...} . # Value. diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 0cae4f69ecdf..996157be95ec 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -57,15 +57,31 @@ proc serve(entity: StoreEntity; turn: Turn; obs: Observe) = facet.run do (turn: Turn): publish(turn, obs.observer.Cap, obs.pattern.capture(initRecord("version", %s)).get) +method serve(entity: StoreEntity; turn: Turn; copy: CopyClosure) = + if not (copy.dest of StoreEntity): + publish(turn, copy.result.Cap, + Error(message: %"destination store is not colocated with source store")) + else: + try: + entity.store.copyClosure(copy.dest.StoreEntity.store, copy.storePath) + publish(turn, copy.result.Cap, ResultOk()) + # TODO: assert some stats or something. + except CatchableError as err: + publish(turn, copy.result.Cap, Error(message: %err.msg)) + method publish(entity: StoreEntity; turn: Turn; a: AssertionRef; h: Handle) = var # orc doesn't handle this as a union object observe: Observe checkPath: CheckStorePath + copyClosure: CopyClosure if checkPath.fromPreserves(a.value): entity.serve(turn, checkPath) elif observe.fromPreserves(a.value): entity.serve(turn, observe) + elif copyClosure.fromPreserves(a.value) and + copyClosure.result of Cap: + entity.serve(turn, copyClosure) else: echo "unhandled assertion ", a.value diff --git a/src/nix_actor/nix_api.nim b/src/nix_actor/nix_api.nim index af6716329db4..a0fd14b331be 100644 --- a/src/nix_actor/nix_api.nim +++ b/src/nix_actor/nix_api.nim @@ -70,6 +70,15 @@ proc isValidPath*(store: Store; path: string): bool = defer: store_path_free(sp) result = nix.store_is_valid_path(store, sp) +proc copyClosure*(src, dst: Store; path: string) = + assert path != "" + mitNix: + let sp = nix.store_parse_path(src, path) + if sp.isNil: + raise newException(CatchableError, "store_parse_path failed") + defer: store_path_free(sp) + nix.store_copy_closure(src, dst, sp) + proc newState*(store: Store; lookupPath: openarray[string]): EvalState = mitNix: var path = allocCStringArray(lookupPath) diff --git a/src/nix_actor/nix_api_store.nim b/src/nix_actor/nix_api_store.nim index 81e95fd89f04..88135fece3a3 100644 --- a/src/nix_actor/nix_api_store.nim +++ b/src/nix_actor/nix_api_store.nim @@ -32,3 +32,4 @@ proc store_realise*(context: NixContext; store: Store; path: StorePath; userdata proc store_get_version*(context: NixContext; store: Store; callback: GetStringCallback; user_data: pointer): nix_err {.nix_api_store, discardable.} +proc store_copy_closure*(context: NixContext; src, dst: Store; path: StorePath): nix_err {.nix_api_store, discardable.} diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index bcd76763d004..5530b5710440 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -84,6 +84,11 @@ type `path`*: string `valid`* {.preservesEmbedded.}: EmbeddedRef + CopyClosure* {.preservesRecord: "copy-closure".} = object + `dest`* {.preservesEmbedded.}: EmbeddedRef + `storePath`*: string + `result`* {.preservesEmbedded.}: EmbeddedRef + StoreResolveStep* {.preservesRecord: "nix-store".} = object `detail`*: StoreResolveDetail @@ -93,6 +98,7 @@ proc `$`*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoStore | StoreResolveDetail | Result | CheckStorePath | + CopyClosure | StoreResolveStep): string = `$`(toPreserves(x)) @@ -102,5 +108,6 @@ proc encode*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoStore | StoreResolveDetail | Result | CheckStorePath | + CopyClosure | StoreResolveStep): seq[byte] = encode(toPreserves(x)) -- cgit 1.4.1 From 6696ba68cbf5ce18a51ed8098dab8ba1935825ed Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 19 Aug 2024 17:11:17 +0300 Subject: Rewrite tests --- test.pr | 110 +++++++++++------------------------------------------ tests/nix-repo.pr | 31 +++++++++++++++ tests/nix-store.pr | 24 ++++++++++++ 3 files changed, 78 insertions(+), 87 deletions(-) create mode 100644 tests/nix-repo.pr create mode 100644 tests/nix-store.pr diff --git a/test.pr b/test.pr index 15150ba2f60e..f62831c4aab1 100755 --- a/test.pr +++ b/test.pr @@ -1,106 +1,42 @@ #!/usr/bin/env -S syndicate-server --control --config -let ?trace = dataspace -$trace +let ?nix-actor = dataspace +$nix-actor ? ?any [ - $log ! - ?- [ $log ! ] + $log ! + ?- [ $log ! ] ] -let ?nixStepDetail = { - import: "" - lookupPath: [ - "nixpkgs=/home/repo/nixpkgs/channel", - ] - store: "auto" - } + $gatekeeper>> + $nix-actor #f> -# > +@"Load test behavior." +> -? > [ - $log ! -] +@"Service instantiation." + -? > -$repo +? [ - $log ! - ? { hello: { meta: { homepage: ?x } } } - [ - $log ! - ] - ? { nim: ?nim } + $log ! + ? > [ - ? { nim: { passthru: ?passthru } } - [ - ? { nim: { passthru: { nim: { buildInputs: [ ?bi ] } } } } - [ - $log ! - ] - ] - ] -] - -let ?nixStoreDetail = { uri: "auto" params: {} } + let ?result = <* $config [ >> + >> + ]> ]> -> -? > -$store -[ - ? ?any - [ - $log ! + $log ! + $nix-actor $result> ] - > - ]>> ] -@"Repo service instantiation." -? > +? > [ - > - ? ?obj> - [ - $log ! - $obj <* $config [ - >> >> - ]> - ]>> - ] -] - -@"Store service instantiation." -? > -[ - > - ? ?obj> - [ - $log ! - $obj <* $config [ - >> - >> - ]> - ]>> - ] + ]> ]> + $nix-actor $result> ] - -@"The authors build system creates this file." - > - ]> -}>> diff --git a/tests/nix-repo.pr b/tests/nix-repo.pr new file mode 100644 index 000000000000..86ef2a47edba --- /dev/null +++ b/tests/nix-repo.pr @@ -0,0 +1,31 @@ +#!/usr/bin/env -S syndicate-server --control --config + +let ?trace = dataspace +$trace +? ?any +[ + $log ! + ?- [ $log ! ] +] + + +let ?detail = { + import: "" + lookupPath: [ + "nixpkgs=/home/repo/nixpkgs/channel", + ] + } + +> +> + +? > +$repo +[ + ? { cowsay: { meta: { homepage: ?x } } } [ + $log ! + ] + ? { shapelib: { meta: { homepage: ?homepage } } } [ + $log ! + ] +] diff --git a/tests/nix-store.pr b/tests/nix-store.pr new file mode 100644 index 000000000000..f71962e1c4f0 --- /dev/null +++ b/tests/nix-store.pr @@ -0,0 +1,24 @@ +#!/usr/bin/env -S syndicate-server --config + +let ?trace = dataspace +$trace +? ?any +[ + $log ! + ?- [ $log ! ] +] + +# > + +? > +$store +[ + ? ?any + [ $log ! ] + + let ?test-path = "/nix/store/c2nszmh6li1r2q5z0aiajlga1rdyqwha-alacritty-0.13.2" + > ]> + > +] -- cgit 1.4.1 From 15371c4f382b202851d0a8130eb01939ef8d778a Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Wed, 21 Aug 2024 14:27:19 +0300 Subject: Add caches to protocol --- protocol.prs | 7 +++++-- sbom.json | 2 +- src/nix_actor/protocol.nim | 24 +++++++++++++++++++++++- test.pr | 14 ++++---------- tests/nix-repo.pr | 7 +++---- tests/nix-store.pr | 2 +- 6 files changed, 37 insertions(+), 19 deletions(-) diff --git a/protocol.prs b/protocol.prs index 19c3dc2ef7a5..f6fa7e76886d 100644 --- a/protocol.prs +++ b/protocol.prs @@ -17,7 +17,7 @@ RepoResolveDetail = { # ] lookupPath: [string ...] -} & @args RepoArgs & @store RepoStore . +} & @args RepoArgs & @store RepoStore & @cacheSpace CacheSpace . # Arguments to call the imported expression with if it is a function. RepoArgs = @present { args: any } / @absent { } . @@ -42,7 +42,10 @@ StoreResolveStep = . StoreResolveDetail = { params: AttrSet uri: string -} . +} & @cacheSpace CacheSpace . + +# A cooperative caching dataspace shared with other Nix stores. +CacheSpace = @cacheSpace { cache: #:any } / @absent { } . CheckStorePath = . diff --git a/sbom.json b/sbom.json index b3542913c928..f0301cc0cf6e 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240819", + "version": "20240821", "authors": [ { "name": "Emery Hemingway" diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 5530b5710440..5ff301da2d09 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -54,6 +54,7 @@ type RepoResolveDetailStore* = Option[Value] `RepoResolveDetail`* {.preservesDictionary.} = object `args`*: Option[Value] + `cache`*: Option[EmbeddedRef] `import`*: string `lookupPath`*: seq[string] `store`*: Option[Value] @@ -62,7 +63,10 @@ type `drvPath`*: string `storePath`*: string - StoreResolveDetail* {.preservesDictionary.} = object + StoreResolveDetailCache* = Option[EmbeddedRef] + StoreResolveDetailUri* = string + `StoreResolveDetail`* {.preservesDictionary.} = object + `cache`*: Option[EmbeddedRef] `params`*: AttrSet `uri`*: string @@ -89,6 +93,22 @@ type `storePath`*: string `result`* {.preservesEmbedded.}: EmbeddedRef + CacheSpaceKind* {.pure.} = enum + `cacheSpace`, `absent` + CacheSpaceCacheSpace* {.preservesDictionary.} = object + `cache`* {.preservesEmbedded.}: EmbeddedRef + + CacheSpaceAbsent* {.preservesDictionary.} = object + + `CacheSpace`* {.preservesOr.} = object + case orKind*: CacheSpaceKind + of CacheSpaceKind.`cacheSpace`: + `cachespace`* {.preservesEmbedded.}: CacheSpaceCacheSpace + + of CacheSpaceKind.`absent`: + `absent`*: CacheSpaceAbsent + + StoreResolveStep* {.preservesRecord: "nix-store".} = object `detail`*: StoreResolveDetail @@ -99,6 +119,7 @@ proc `$`*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoStore | Result | CheckStorePath | CopyClosure | + CacheSpace | StoreResolveStep): string = `$`(toPreserves(x)) @@ -109,5 +130,6 @@ proc encode*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoStore | Result | CheckStorePath | CopyClosure | + CacheSpace | StoreResolveStep): seq[byte] = encode(toPreserves(x)) diff --git a/test.pr b/test.pr index f62831c4aab1..23ba00902917 100755 --- a/test.pr +++ b/test.pr @@ -11,10 +11,13 @@ $nix-actor $gatekeeper>> $nix-actor #f> +let ?nix-cache = dataspace + @"Load test behavior." -> +> @"Service instantiation." + ? @@ -31,12 +34,3 @@ $nix-actor $nix-actor $result> ] ] - -? > -[ - let ?result = <* $config [ >> - >> - ]> ]> - $nix-actor $result> -] diff --git a/tests/nix-repo.pr b/tests/nix-repo.pr index 86ef2a47edba..ead950f29ea8 100644 --- a/tests/nix-repo.pr +++ b/tests/nix-repo.pr @@ -8,15 +8,14 @@ $trace ?- [ $log ! ] ] - let ?detail = { + # store: $store + cache: $nix-cache import: "" lookupPath: [ - "nixpkgs=/home/repo/nixpkgs/channel", + "nixpkgs=/home/repo/nixpkgs/channel" ] } - -> > ? > diff --git a/tests/nix-store.pr b/tests/nix-store.pr index f71962e1c4f0..3cbf668fdec5 100644 --- a/tests/nix-store.pr +++ b/tests/nix-store.pr @@ -8,7 +8,7 @@ $trace ?- [ $log ! ] ] -# > +> ? > $store -- cgit 1.4.1 From 3cced46db25d6c0040e018d9d2064a3df46342d8 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Thu, 22 Aug 2024 15:47:13 +0300 Subject: Add realisation to nix-repo --- protocol.prs | 17 ++++--- sbom.json | 2 +- src/nix_actor.nim | 103 +++++++++++++++++++++++++++------------- src/nix_actor/nix_api_store.nim | 2 + src/nix_actor/nix_values.nim | 61 +++++++++++++++++++----- src/nix_actor/protocol.nim | 17 +++++-- tests/nix-repo.pr | 5 +- tests/test.nim | 15 +++--- 8 files changed, 154 insertions(+), 68 deletions(-) diff --git a/protocol.prs b/protocol.prs index f6fa7e76886d..c214dcf80312 100644 --- a/protocol.prs +++ b/protocol.prs @@ -28,15 +28,6 @@ RepoStore = @uri { store: string } / @cap { store: #:any } / @absent { } . # Common error type. Error = . -# Represents a Nix derivation. -# The @storePath can be realized as store object from @drvPath. -# -# If an attrset value resulting from evaluation has a "drvPath" attribute -# then a drv record is returned in place of the attrset. Returning the -# attrset is not feasible because otherwise lazy values would explode into -# complete dependency trees. -Derivation = . - # Gatekeeper step to access a Nix store. StoreResolveStep = . StoreResolveDetail = { @@ -52,6 +43,14 @@ CheckStorePath = . # Assertion. The store that this asserted to will copy the closure of @storePath to @destination. When the copy completes or fails a Result value is asserted to @result. CopyClosure = . +# Represents a Nix derivation. The @value can be realised via @context. +Derivation = . + +# Assertion. +Realise = . + +Context = #:any . + AttrSet = {symbol: any ...:...} . # Value. diff --git a/sbom.json b/sbom.json index f0301cc0cf6e..463e295019e6 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240821", + "version": "20240822", "authors": [ { "name": "Emery Hemingway" diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 996157be95ec..2302a4c61596 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -2,12 +2,12 @@ # SPDX-License-Identifier: Unlicense import - std/[options, strutils, tables, times], + std/[options, strutils, tables], pkg/preserves, pkg/preserves/sugar, pkg/syndicate, pkg/syndicate/[gatekeepers, patterns, relays], - ./nix_actor/[nix_api, nix_api_store, nix_values], + ./nix_actor/[nix_api, nix_values], ./nix_actor/protocol proc echo(args: varargs[string, `$`]) {.used.} = @@ -15,8 +15,19 @@ proc echo(args: varargs[string, `$`]) {.used.} = type Value = preserves.Value +template tryPublish(turn: Turn, cap: Cap; body: untyped) = + try: body + except CatchableError as err: + publish(turn, cap, Error(message: %err.msg)) + +proc publishOk(turn: Turn; cap: Cap, v: Value) = + publish(turn, cap, ResultOk(value: v)) + +proc publishError(turn: Turn; cap: Cap, v: Value) = + publish(turn, cap, Error(message: v)) + type - StoreEntity = ref object of Entity + StoreEntity {.final.} = ref object of Entity self: Cap store: Store @@ -40,11 +51,9 @@ proc newStoreEntity(turn: Turn; detail: StoreResolveDetail): StoreEntity = entity proc serve(entity: StoreEntity; turn: Turn; checkPath: CheckStorePath) = - try: + tryPublish(turn, checkPath.valid.Cap): let v = entity.store.isValidPath(checkPath.path) publish(turn, checkPath.valid.Cap, initRecord("ok", %v)) - except CatchableError as err: - publish(turn, checkPath.valid.Cap, Error(message: %err.msg)) proc serve(entity: StoreEntity; turn: Turn; obs: Observe) = let facet = turn.facet @@ -62,12 +71,10 @@ method serve(entity: StoreEntity; turn: Turn; copy: CopyClosure) = publish(turn, copy.result.Cap, Error(message: %"destination store is not colocated with source store")) else: - try: + tryPublish(turn, copy.result.Cap): entity.store.copyClosure(copy.dest.StoreEntity.store, copy.storePath) publish(turn, copy.result.Cap, ResultOk()) # TODO: assert some stats or something. - except CatchableError as err: - publish(turn, copy.result.Cap, Error(message: %err.msg)) method publish(entity: StoreEntity; turn: Turn; a: AssertionRef; h: Handle) = var @@ -86,9 +93,9 @@ method publish(entity: StoreEntity; turn: Turn; a: AssertionRef; h: Handle) = echo "unhandled assertion ", a.value type - RepoEntity = ref object of Entity + RepoEntity {.final.} = ref object of Entity self: Cap - store: Store + store: StoreEntity state: EvalState root: NixValue @@ -97,19 +104,19 @@ proc newRepoEntity(turn: Turn; detail: RepoResolveDetail): RepoEntity = turn.onStop do (turn: Turn): if not entity.state.isNil: entity.state.close() - if not entity.store.isNil: - entity.store.close() if detail.store.isSome: var other = detail.store.get.unembed(StoreEntity) if other.isSome: - entity.store = other.get.store + entity.store = other.get elif detail.store.get.isString: - entity.store = openStore(detail.store.get.string) + var storeDetail = StoreResolveDetail(cache: detail.cache, uri: detail.store.get.string) + entity.store = newStoreEntity(turn, storeDetail) else: - raise newException(CatchableError, "invalid store parameter") + raise newException(CatchableError, "invalid store parameter for nix-repo: " & $detail.store.get) else: - entity.store = openStore() - entity.state = newState(entity.store, detail.lookupPath) + var storeDetail = StoreResolveDetail(cache: detail.cache, uri: "auto") + entity.store = newStoreEntity(turn, storeDetail) + entity.state = newState(entity.store.store, detail.lookupPath) entity.root = entity.state.evalFromString("import " & detail.`import`, "") if detail.args.isSome: var na = detail.args.get.toNix(entity.state) @@ -117,27 +124,55 @@ proc newRepoEntity(turn: Turn; detail: RepoResolveDetail): RepoEntity = entity.self = newCap(turn, entity) entity +proc serve(repo: RepoEntity; turn: Turn; obs: Observe) = + var + analysis = analyse(obs.pattern) + captures = newSeq[Value](analysis.capturePaths.len) + block stepping: + for i, path in analysis.constPaths: + var v = repo.state.step(repo.root, path) + if v.isNone or v.get != analysis.constValues[i]: + let null = initRecord("null") + for v in captures.mitems: v = null + break stepping + for i, path in analysis.capturePaths: + var v = repo.state.step(repo.root, path) + if v.isSome: + captures[i] = turn.facet.exportNix(v.get) + else: captures[i] = initRecord("null") + discard publish(turn, Cap obs.observer, captures) + +proc serve(repo: RepoEntity; turn: Turn; r: Realise) = + tryPublish(turn, r.result.Cap): + var drv: Derivation + if not drv.fromPreserves(r.value): + publishError(turn, r.result.Cap, %("failed to parse Derivation: " & $r.value)) + else: + var dummyCap = drv.context.unembed(Cap) + if dummyCap.isNone: + publishError(turn, r.result.Cap, %"derivation context is not a Cap") + else: + if not(dummyCap.get.target of NixValueRef): + publishError(turn, r.result.Cap, %"derivation context is not a NixValueRef") + else: + let v = repo.state.realise(dummyCap.get.target.NixValueRef.value) + publishOk(turn, r.result.Cap, v) + # TODO: this is awkward. + method publish(repo: RepoEntity; turn: Turn; a: AssertionRef; h: Handle) = ## Respond to observations with dataspace semantics, minus retraction ## of assertions in response to the retraction of observations. ## This entity is scoped to immutable data so this shouldn't be a problem. - var obs: Observe + var + obs: Observe + realise: Realise if obs.fromPreserves(a.value) and obs.observer of Cap: - var analysis = analyse(obs.pattern) - var captures = newSeq[Value](analysis.capturePaths.len) - block stepping: - for i, path in analysis.constPaths: - var v = repo.state.step(repo.root, path) - if v.isNone or v.get != analysis.constValues[i]: - let null = initRecord("null") - for v in captures.mitems: v = null - break stepping - for i, path in analysis.capturePaths: - var v = repo.state.step(repo.root, path) - if v.isSome: - captures[i] = v.get.unthunkAll - else: captures[i] = initRecord("null") - discard publish(turn, Cap obs.observer, captures) + serve(repo, turn, obs) + elif realise.fromPreserves(a.value) and realise.result of Cap: + serve(repo, turn, realise) + else: + when not defined(release): + echo "unhandled assertion ", a.value proc main() = initLibstore() diff --git a/src/nix_actor/nix_api_store.nim b/src/nix_actor/nix_api_store.nim index 88135fece3a3..ef4ef525866f 100644 --- a/src/nix_actor/nix_api_store.nim +++ b/src/nix_actor/nix_api_store.nim @@ -2,6 +2,8 @@ import ./nix_api_types +{.passL: "-L/home/repo/nix/src/libstore-c".} + {.pragma: nix_api_store, header: "nix_api_store.h", importc: "nix_$1".} proc libstore_init*(context: NixContext): nix_err {.nix_api_store, discardable.} diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim index 9b2d6be18460..6f5762d1d3a1 100644 --- a/src/nix_actor/nix_values.nim +++ b/src/nix_actor/nix_values.nim @@ -4,11 +4,15 @@ import std/options, pkg/preserves, - ./[nix_api, nix_api_util, nix_api_value, utils] + pkg/syndicate/actors, + ./[nix_api, nix_api_util, nix_api_value, protocol, utils] type Value = preserves.Value NixValue* = nix_api.Value + NixValueRef* {.final.} = ref object of Entity + value*: NixValue + StringThunkRef = ref StringThunkObj StringThunkObj = object of EmbeddedObj data: Option[string] @@ -21,15 +25,32 @@ proc thunkString(start: cstring; n: cuint; state: pointer) {.cdecl.} = copyMem(buf[0].addr, start, buf.len) thunk.data = buf.move.some -proc unthunk*(v: Value): Value = +proc unthunk(v: Value): Value = let thunk = v.unembed(StringThunkRef) - assert thunk.isSome - assert thunk.get.data.isSome - thunk.get.data.get.toPreserves + result = + if thunk.isSome and thunk.get.data.isSome: + thunk.get.data.get.toPreserves + else: v -proc unthunkAll*(v: Value): Value = +proc unthunkAll(v: Value): Value = v.mapEmbeds(unthunk) +proc exportNix*(facet: Facet; v: Value): Value = + proc op(v: Value): Value = + result = + if v.kind != pkEmbedded: v + else: + if v.embeddedRef of StringThunkRef: + var thunk = v.embeddedRef.StringThunkRef + if thunk.data.isSome: + thunk.data.get.toPreserves + else: v + elif v.embeddedRef of NixValueRef: + facet.newCap(v.embeddedRef.NixValueRef).embed + else: + v + v.mapEmbeds(op) + proc callThru(state: EvalState; nv: NixValue): NixValue = result = nv mitNix: @@ -76,7 +97,11 @@ proc toPreserves*(state: EvalState; value: NixValue; nix: NixContext): Value {.g str = state.apply(str, value) result = state.toPreserves(str, nix) elif nix.has_attr_byname(value, state, "outPath"): - result = state.toPreserves(nix.get_attr_byname(value, state, "outPath"), nix) + var outPath = nix.get_attr_byname(value, state, "outPath") + result = Derivation( + value: state.toPreserves(outPath, nix), + context: NixValueRef(value: value).embed, + ).toPreserves else: let n = nix.getAttrsSize(value) result = initDictionary(int n) @@ -123,11 +148,13 @@ proc translate*(nix: NixContext; state: EvalState; pr: preserves.Value): NixValu if pr.isRecord("null", 0): nix.init_null(result) elif pr.isRecord("drv", 2): - let b = nix.make_bindings_builder(state, 2) - defer: bindings_builder_free(b) - nix.bindings_builder_insert(b, "drvPath", nix.translate(state, pr.fields[0])) - nix.bindings_builder_insert(b, "outPath", nix.translate(state, pr.fields[1])) - nix.make_attrs(result, b) + var drv: Derivation + if not drv.fromPreserves(pr): + raise newException(ValueError, "invalid derivation: " & $pr) + var nixValRef = drv.context.unembed(NixValueRef) + if not nixValRef.isSome: + raise newException(ValueError, "invalid Nix context: " & $drv.context) + result = nixValRef.get.value else: raise newException(ValueError, "cannot convert Preserves record to Nix: " & $pr) of pkSequence, pkSet: @@ -186,3 +213,13 @@ proc step*(state: EvalState; nv: NixValue; path: openarray[preserves.Value]): Op raiseAssert("cannot step " & $kind) return result = state.toPreserves(nv).some + +proc realise*(nix: NixContext; state: EvalState; val: NixValue): Value = + result = "".toPreserves + var rs = nix.string_realise(state, val, false) + result.string = newString(realised_string_get_buffer_size(rs)) + copyMem(result.string[0].addr, realised_string_get_buffer_start(rs), result.string.len) + realised_string_free(rs) + +proc realise*(state: EvalState; val: NixValue): Value = + mitNix: result = nix.realise(state, val) diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 5ff301da2d09..3b3adf7086c3 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -26,6 +26,10 @@ type `detail`*: RepoResolveDetail AttrSet* = Table[Symbol, Value] + Realise* {.preservesRecord: "realise".} = object + `value`*: Value + `result`* {.preservesEmbedded.}: EmbeddedRef + RepoStoreKind* {.pure.} = enum `uri`, `cap`, `absent` RepoStoreUri* {.preservesDictionary.} = object @@ -49,6 +53,7 @@ type RepoResolveDetailArgs* = Option[Value] + RepoResolveDetailCache* = Option[EmbeddedRef] RepoResolveDetailImport* = string RepoResolveDetailLookupPath* = seq[string] RepoResolveDetailStore* = Option[Value] @@ -60,8 +65,8 @@ type `store`*: Option[Value] Derivation* {.preservesRecord: "drv".} = object - `drvPath`*: string - `storePath`*: string + `value`*: Value + `context`*: Value StoreResolveDetailCache* = Option[EmbeddedRef] StoreResolveDetailUri* = string @@ -84,6 +89,7 @@ type `ok`*: ResultOk + Context* = EmbeddedRef CheckStorePath* {.preservesRecord: "check-path".} = object `path`*: string `valid`* {.preservesEmbedded.}: EmbeddedRef @@ -112,22 +118,25 @@ type StoreResolveStep* {.preservesRecord: "nix-store".} = object `detail`*: StoreResolveDetail -proc `$`*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoStore | +proc `$`*(x: Error | RepoArgs | RepoResolveStep | AttrSet | Realise | RepoStore | RepoResolveDetail | Derivation | StoreResolveDetail | Result | + Context | CheckStorePath | CopyClosure | CacheSpace | StoreResolveStep): string = `$`(toPreserves(x)) -proc encode*(x: Error | RepoArgs | RepoResolveStep | AttrSet | RepoStore | +proc encode*(x: Error | RepoArgs | RepoResolveStep | AttrSet | Realise | + RepoStore | RepoResolveDetail | Derivation | StoreResolveDetail | Result | + Context | CheckStorePath | CopyClosure | CacheSpace | diff --git a/tests/nix-repo.pr b/tests/nix-repo.pr index ead950f29ea8..035868dd4a7f 100644 --- a/tests/nix-repo.pr +++ b/tests/nix-repo.pr @@ -21,8 +21,9 @@ let ?detail = { ? > $repo [ - ? { cowsay: { meta: { homepage: ?x } } } [ - $log ! + ? { cowsay: ?pkg } [ + $log ! + > ]>> ] ? { shapelib: { meta: { homepage: ?homepage } } } [ $log ! diff --git a/tests/test.nim b/tests/test.nim index aa75842b50a1..b81049e9efdf 100644 --- a/tests/test.nim +++ b/tests/test.nim @@ -5,6 +5,7 @@ import std/[unittest], pkg/balls, pkg/preserves, + pkg/syndicate, ../src/nix_actor/[nix_api, nix_values, protocol] suite "libexpr": @@ -15,12 +16,14 @@ suite "libexpr": state = newState(store, []) proc checkConversion(s: string) = - var nixVal = state.evalFromString(s, "") - state.force(nixVal) - nixVal.close() - var pr = state.toPreserves(nixVal) - pr = pr.unthunkAll - echo pr + runActor("checkConversion") do (turn: Turn): + var nixVal = state.evalFromString(s, "") + state.force(nixVal) + nixVal.close() + var pr = state.toPreserves(nixVal) + checkpoint pr + var wirePr = turn.facet.exportNix(pr) + checkpoint wirePr test "lists": let samples = [ -- cgit 1.4.1 From a727040cca3034258ba452baee8c20677b4ea433 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Thu, 22 Aug 2024 15:49:25 +0300 Subject: balls: seperate runs for test modules --- tests/Tupfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tupfile b/tests/Tupfile index 49d1bc1be290..09dc80a8dd72 100644 --- a/tests/Tupfile +++ b/tests/Tupfile @@ -1,2 +1,2 @@ include_rules -: t*.nim |> !balls |> +: foreach t*.nim |> !balls |> -- cgit 1.4.1 From 35ff31256d33a8c24589a45dcd3483025e1a19b7 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 19 Aug 2024 17:10:59 +0300 Subject: Add copy-closure to nix-store --- protocol.prs | 6 +++--- src/nix_actor.nim | 19 ++++++++++++------- tests/nix-store.pr | 25 +++++++++++++++++++------ 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/protocol.prs b/protocol.prs index c214dcf80312..147611860dcb 100644 --- a/protocol.prs +++ b/protocol.prs @@ -40,9 +40,6 @@ CacheSpace = @cacheSpace { cache: #:any } / @absent { } . CheckStorePath = . -# Assertion. The store that this asserted to will copy the closure of @storePath to @destination. When the copy completes or fails a Result value is asserted to @result. -CopyClosure = . - # Represents a Nix derivation. The @value can be realised via @context. Derivation = . @@ -51,6 +48,9 @@ Realise = . Context = #:any . +# Assertion. The store that this asserted to will copy the closure of @storePath to @destination. When the copy completes or fails a Result value is asserted to @result. +CopyClosure = . + AttrSet = {symbol: any ...:...} . # Value. diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 2302a4c61596..997f90a80dca 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -26,6 +26,10 @@ proc publishOk(turn: Turn; cap: Cap, v: Value) = proc publishError(turn: Turn; cap: Cap, v: Value) = publish(turn, cap, Error(message: v)) +proc unembedEntity(emb: EmbeddedRef; E: typedesc): Option[E] = + if emb of Cap and emb.Cap.target of E: + result = emb.Cap.target.E.some + type StoreEntity {.final.} = ref object of Entity self: Cap @@ -67,13 +71,13 @@ proc serve(entity: StoreEntity; turn: Turn; obs: Observe) = publish(turn, obs.observer.Cap, obs.pattern.capture(initRecord("version", %s)).get) method serve(entity: StoreEntity; turn: Turn; copy: CopyClosure) = - if not (copy.dest of StoreEntity): - publish(turn, copy.result.Cap, - Error(message: %"destination store is not colocated with source store")) + var dest = copy.dest.unembedEntity(StoreEntity) + if dest.isNone: + publishError(turn, copy.result.Cap, %"destination store is not colocated with source store") else: tryPublish(turn, copy.result.Cap): - entity.store.copyClosure(copy.dest.StoreEntity.store, copy.storePath) - publish(turn, copy.result.Cap, ResultOk()) + entity.store.copyClosure(dest.get.store, copy.storePath) + publishOk(turn, copy.result.Cap, %true) # TODO: assert some stats or something. method publish(entity: StoreEntity; turn: Turn; a: AssertionRef; h: Handle) = @@ -90,7 +94,8 @@ method publish(entity: StoreEntity; turn: Turn; a: AssertionRef; h: Handle) = copyClosure.result of Cap: entity.serve(turn, copyClosure) else: - echo "unhandled assertion ", a.value + when not defined(release): + echo "nix-store: unhandled assertion ", a.value type RepoEntity {.final.} = ref object of Entity @@ -172,7 +177,7 @@ method publish(repo: RepoEntity; turn: Turn; a: AssertionRef; h: Handle) = serve(repo, turn, realise) else: when not defined(release): - echo "unhandled assertion ", a.value + echo "nix-repo: unhandled assertion ", a.value proc main() = initLibstore() diff --git a/tests/nix-store.pr b/tests/nix-store.pr index 3cbf668fdec5..f22d30040792 100644 --- a/tests/nix-store.pr +++ b/tests/nix-store.pr @@ -8,17 +8,30 @@ $trace ?- [ $log ! ] ] -> +let ?default-detail = { uri: "auto" params: {} cache: $nix-cache } +let ?test-detail = { label: "test-store" uri: "file:///tmp/nix-test-store" params: {} } -? > -$store +> + +? > [ - ? ?any + $default-store ? ?any [ $log ! ] - let ?test-path = "/nix/store/c2nszmh6li1r2q5z0aiajlga1rdyqwha-alacritty-0.13.2" - > ]> > + + > + ? > + [ + $default-store += > ]> + > + ] + ] -- cgit 1.4.1 From 8f810abca5d124d38802b78538c7cfc66c38246e Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sat, 24 Aug 2024 13:20:32 +0300 Subject: Fix assertion of store caps --- sbom.json | 2 +- src/nix_actor.nim | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/sbom.json b/sbom.json index 463e295019e6..8c1fdb9fa2e2 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240822", + "version": "20240824", "authors": [ { "name": "Emery Hemingway" diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 997f90a80dca..934a69f6dced 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -30,6 +30,10 @@ proc unembedEntity(emb: EmbeddedRef; E: typedesc): Option[E] = if emb of Cap and emb.Cap.target of E: result = emb.Cap.target.E.some +proc unembedEntity(v: Value; E: typedesc): Option[E] = + if v.isEmbeddedRef: + result = v.embeddedRef.unembedEntity(E) + type StoreEntity {.final.} = ref object of Entity self: Cap @@ -70,7 +74,7 @@ proc serve(entity: StoreEntity; turn: Turn; obs: Observe) = facet.run do (turn: Turn): publish(turn, obs.observer.Cap, obs.pattern.capture(initRecord("version", %s)).get) -method serve(entity: StoreEntity; turn: Turn; copy: CopyClosure) = +proc serve(entity: StoreEntity; turn: Turn; copy: CopyClosure) = var dest = copy.dest.unembedEntity(StoreEntity) if dest.isNone: publishError(turn, copy.result.Cap, %"destination store is not colocated with source store") @@ -110,7 +114,7 @@ proc newRepoEntity(turn: Turn; detail: RepoResolveDetail): RepoEntity = if not entity.state.isNil: entity.state.close() if detail.store.isSome: - var other = detail.store.get.unembed(StoreEntity) + var other = detail.store.get.unembedEntity(StoreEntity) if other.isSome: entity.store = other.get elif detail.store.get.isString: -- cgit 1.4.1 From 9349f9bcf9998831315577ce8794f50a9359f62a Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sat, 24 Aug 2024 13:22:08 +0300 Subject: Fix pkg-config importing --- src/nix_actor/nix_api.nim | 3 --- src/nix_actor/nix_api_expr.nim | 3 +++ src/nix_actor/nix_api_store.nim | 3 ++- tup.config.nix | 16 ++++++++++++++++ 4 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 tup.config.nix diff --git a/src/nix_actor/nix_api.nim b/src/nix_actor/nix_api.nim index a0fd14b331be..1f36fc9edfd0 100644 --- a/src/nix_actor/nix_api.nim +++ b/src/nix_actor/nix_api.nim @@ -11,9 +11,6 @@ import export NixContext, Store, EvalState, Value, ValueType, gc_decref, isNil -{.passC: staticExec"$PKG_CONFIG --cflags nix-expr-c".} -{.passL: staticExec"$PKG_CONFIG --libs nix-expr-c".} - type StringCallback = proc (s: string) {.closure.} StringCallbackState = object diff --git a/src/nix_actor/nix_api_expr.nim b/src/nix_actor/nix_api_expr.nim index 8bb42c3d9349..748933ad46fe 100644 --- a/src/nix_actor/nix_api_expr.nim +++ b/src/nix_actor/nix_api_expr.nim @@ -2,6 +2,9 @@ import ./nix_api_types +{.passC: staticExec"$PKG_CONFIG --cflags nix-expr-c".} +{.passL: staticExec"$PKG_CONFIG --libs nix-expr-c".} + {.pragma: nix_api_expr, header: "nix_api_expr.h", importc: "nix_$1".} proc libexpr_init*(context: NixContext): nix_err {.nix_api_expr.} diff --git a/src/nix_actor/nix_api_store.nim b/src/nix_actor/nix_api_store.nim index ef4ef525866f..0e1719062248 100644 --- a/src/nix_actor/nix_api_store.nim +++ b/src/nix_actor/nix_api_store.nim @@ -2,7 +2,8 @@ import ./nix_api_types -{.passL: "-L/home/repo/nix/src/libstore-c".} +{.passC: staticExec"$PKG_CONFIG --cflags nix-store-c".} +{.passL: staticExec"$PKG_CONFIG --libs nix-store-c".} {.pragma: nix_api_store, header: "nix_api_store.h", importc: "nix_$1".} diff --git a/tup.config.nix b/tup.config.nix new file mode 100644 index 000000000000..892248b60eba --- /dev/null +++ b/tup.config.nix @@ -0,0 +1,16 @@ +{ lib, pkgs, ... }: + +let + nix' = pkgs.nixVersions.latest.overrideAttrs (_: { + version = "2024-08-23"; + src = pkgs.fetchFromGitHub { + owner = "nixos"; + repo = "nix"; + rev = "85f1aa6b3df5c5fcc924a74e2a9cc8acea9ba0e1"; + hash = "sha256-3+UgAktTtkGUNpxMxr+q+R+z3r026L3PwJzG6RD2IXM="; + }; + }); +in +{ + PKG_CONFIG_PATH_nix = "${lib.getDev nix'}/lib/pkgconfig"; +} -- cgit 1.4.1 From c5b315914f95871c85ed3d30caf9f07f98549cb1 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sat, 24 Aug 2024 13:22:51 +0300 Subject: Add eval assertion --- protocol.prs | 6 ++++++ src/nix_actor.nim | 15 ++++++++++++++- src/nix_actor/nix_api.nim | 8 +++++++- src/nix_actor/nix_values.nim | 2 +- src/nix_actor/protocol.nim | 10 ++++++++-- 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/protocol.prs b/protocol.prs index 147611860dcb..2852ff103cc9 100644 --- a/protocol.prs +++ b/protocol.prs @@ -51,6 +51,12 @@ Context = #:any . # Assertion. The store that this asserted to will copy the closure of @storePath to @destination. When the copy completes or fails a Result value is asserted to @result. CopyClosure = . +# Assertion. +# Eval at a nix-repo. @expr must be a function that takes two parameters, +# the first is the nix-repo value, the second is @args. +# The result is asserted to @result +Eval = . + AttrSet = {symbol: any ...:...} . # Value. diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 934a69f6dced..4e4bde699541 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -126,7 +126,7 @@ proc newRepoEntity(turn: Turn; detail: RepoResolveDetail): RepoEntity = var storeDetail = StoreResolveDetail(cache: detail.cache, uri: "auto") entity.store = newStoreEntity(turn, storeDetail) entity.state = newState(entity.store.store, detail.lookupPath) - entity.root = entity.state.evalFromString("import " & detail.`import`, "") + entity.root = entity.state.evalFromString("import " & detail.`import`) if detail.args.isSome: var na = detail.args.get.toNix(entity.state) entity.root = entity.state.apply(entity.root, na) @@ -168,6 +168,15 @@ proc serve(repo: RepoEntity; turn: Turn; r: Realise) = publishOk(turn, r.result.Cap, v) # TODO: this is awkward. +proc serve(repo: RepoEntity; turn: Turn; eval: Eval) = + tryPublish(turn, eval.result.Cap): + var + expr = repo.state.evalFromString(eval.expr) + args = eval.args.toNix(repo.state) + val = repo.state.call(expr, repo.root, args) + res = repo.state.toPreserves(val) + publishOk(turn, eval.result.Cap, res) + method publish(repo: RepoEntity; turn: Turn; a: AssertionRef; h: Handle) = ## Respond to observations with dataspace semantics, minus retraction ## of assertions in response to the retraction of observations. @@ -175,10 +184,13 @@ method publish(repo: RepoEntity; turn: Turn; a: AssertionRef; h: Handle) = var obs: Observe realise: Realise + eval: Eval if obs.fromPreserves(a.value) and obs.observer of Cap: serve(repo, turn, obs) elif realise.fromPreserves(a.value) and realise.result of Cap: serve(repo, turn, realise) + elif eval.fromPreserves(a.value) and eval.result of Cap: + serve(repo, turn, eval) else: when not defined(release): echo "nix-repo: unhandled assertion ", a.value @@ -196,5 +208,6 @@ proc main() = gk.serve do (turn: Turn; step: RepoResolveStep) -> Resolved: newRepoEntity(turn, step.detail).self.resolveAccepted + # TODO: support for stepping through attrs? main() diff --git a/src/nix_actor/nix_api.nim b/src/nix_actor/nix_api.nim index 1f36fc9edfd0..656217461ac0 100644 --- a/src/nix_actor/nix_api.nim +++ b/src/nix_actor/nix_api.nim @@ -92,7 +92,7 @@ proc close*(value: Value) = proc evalFromString*(nix: NixContext; state: EvalState; expr, path: string; result: Value) = discard nix.expr_eval_from_string(state, expr, path, result) -proc evalFromString*(state: EvalState; expr, path: string): Value = +proc evalFromString*(state: EvalState; expr: string; path = ""): Value = mitNix: try: result = nix.alloc_value(state) @@ -116,3 +116,9 @@ proc apply(nix: NixContext; state: EvalState; fn, arg: Value): Value = proc apply*(state: EvalState; fn, arg: Value): Value = mitNix: result = nix.apply(state, fn, arg) + +proc call*(state: EvalState; fn: Value; args: varargs[Value]): Value = + mitNix: + result = nix.alloc_value(state) + var array = cast[ptr UncheckedArray[Value]](args) + discard nix.value_call_multi(state, fn, args.len.csize_t, array, result) diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim index 6f5762d1d3a1..e97d19974daa 100644 --- a/src/nix_actor/nix_values.nim +++ b/src/nix_actor/nix_values.nim @@ -38,7 +38,7 @@ proc unthunkAll(v: Value): Value = proc exportNix*(facet: Facet; v: Value): Value = proc op(v: Value): Value = result = - if v.kind != pkEmbedded: v + if not v.isEmbeddedRef: v else: if v.embeddedRef of StringThunkRef: var thunk = v.embeddedRef.StringThunkRef diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 3b3adf7086c3..6f8755e7d986 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -6,6 +6,11 @@ type Error* {.preservesRecord: "error".} = object `message`*: Value + Eval* {.preservesRecord: "eval".} = object + `expr`*: string + `args`*: Value + `result`* {.preservesEmbedded.}: EmbeddedRef + RepoArgsKind* {.pure.} = enum `present`, `absent` RepoArgsPresent* {.preservesDictionary.} = object @@ -118,7 +123,8 @@ type StoreResolveStep* {.preservesRecord: "nix-store".} = object `detail`*: StoreResolveDetail -proc `$`*(x: Error | RepoArgs | RepoResolveStep | AttrSet | Realise | RepoStore | +proc `$`*(x: Error | Eval | RepoArgs | RepoResolveStep | AttrSet | Realise | + RepoStore | RepoResolveDetail | Derivation | StoreResolveDetail | @@ -130,7 +136,7 @@ proc `$`*(x: Error | RepoArgs | RepoResolveStep | AttrSet | Realise | RepoStore StoreResolveStep): string = `$`(toPreserves(x)) -proc encode*(x: Error | RepoArgs | RepoResolveStep | AttrSet | Realise | +proc encode*(x: Error | Eval | RepoArgs | RepoResolveStep | AttrSet | Realise | RepoStore | RepoResolveDetail | Derivation | -- cgit 1.4.1 From a016dc9d273cfb4e7fc6d5a51e358b06c0de28ac Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sat, 24 Aug 2024 13:23:14 +0300 Subject: Update tests --- test.pr | 44 ++++++++++++++++++++++++++++++++++++-------- tests/nix-repo.pr | 36 +++++++++++++++++------------------- tests/nix-store.pr | 8 ++++---- 3 files changed, 57 insertions(+), 31 deletions(-) diff --git a/test.pr b/test.pr index 23ba00902917..8d6e6930d066 100755 --- a/test.pr +++ b/test.pr @@ -1,20 +1,48 @@ #!/usr/bin/env -S syndicate-server --control --config let ?nix-actor = dataspace -$nix-actor -? ?any -[ - $log ! - ?- [ $log ! ] -] + +# $nix-actor +# ? ?any +# [ +# $log ! +# ?- [ $log ! ] +# ] $gatekeeper>> $nix-actor #f> let ?nix-cache = dataspace -@"Load test behavior." -> +let ?parent-route = ] + > + +let ?resolve-parent = + + +$log ! }> + +? > [ + $control ! +] + +? >> +[ + $parent += > + $parent + ? > + [ + @"Load test behavior." + $config += > + ] +] @"Service instantiation." diff --git a/tests/nix-repo.pr b/tests/nix-repo.pr index 035868dd4a7f..f012c12f0189 100644 --- a/tests/nix-repo.pr +++ b/tests/nix-repo.pr @@ -1,30 +1,28 @@ #!/usr/bin/env -S syndicate-server --control --config -let ?trace = dataspace -$trace -? ?any +? > [ - $log ! - ?- [ $log ! ] + let ?detail = { + store: $store + cache: $nix-cache + import: "" + lookupPath: [ + "nixpkgs=/home/repo/nixpkgs/channel" + ] + } + <-require-service > ] -let ?detail = { - # store: $store - cache: $nix-cache - import: "" - lookupPath: [ - "nixpkgs=/home/repo/nixpkgs/channel" - ] - } -> - ? > -$repo [ - ? { cowsay: ?pkg } [ - $log ! - > ]>> + $repo + ? { plan9port: ?pkg } [ + $notifier + + $log ! + > ]>> ] + $repo ? { shapelib: { meta: { homepage: ?homepage } } } [ $log ! ] diff --git a/tests/nix-store.pr b/tests/nix-store.pr index f22d30040792..4b2e145c22a0 100644 --- a/tests/nix-store.pr +++ b/tests/nix-store.pr @@ -9,9 +9,9 @@ $trace ] let ?default-detail = { uri: "auto" params: {} cache: $nix-cache } -let ?test-detail = { label: "test-store" uri: "file:///tmp/nix-test-store" params: {} } +let ?test-detail = { label: "test" uri: "file:///tmp/nix-test-store" params: {} } -> +# > ? > [ @@ -27,10 +27,10 @@ let ?test-detail = { label: "test-store" uri: "file:///tmp/nix-test-store" param > ? > [ - $default-store += > ]> + <* $notifier [ > ]> > ] -- cgit 1.4.1 From 84701460a7dccba6c203a24fc5b4866ca51ddef9 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sun, 25 Aug 2024 20:41:15 +0300 Subject: Stop discarding serious errors, reörder toPreserves args MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sbom.json | 2 +- src/nix_actor/nix_api.nim | 32 +++++++-------- src/nix_actor/nix_api_store.nim | 14 +++---- src/nix_actor/nix_api_value.nim | 30 +++++++------- src/nix_actor/nix_values.nim | 87 +++++++++++++++++++++++------------------ src/nix_actor/utils.nim | 6 ++- 6 files changed, 91 insertions(+), 80 deletions(-) diff --git a/sbom.json b/sbom.json index 8c1fdb9fa2e2..4082821204a3 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240824", + "version": "20240825", "authors": [ { "name": "Emery Hemingway" diff --git a/src/nix_actor/nix_api.nim b/src/nix_actor/nix_api.nim index 656217461ac0..e382602ca8c1 100644 --- a/src/nix_actor/nix_api.nim +++ b/src/nix_actor/nix_api.nim @@ -26,11 +26,11 @@ proc receiveString(start: cstring; n: cuint; state: pointer) {.cdecl.} = proc initLibstore*() = mitNix: - discard nix.libstore_init() + checkError nix.libstore_init() proc initLibexpr*() = mitNix: - discard nix.libexpr_init() + checkError nix.libexpr_init() proc openStore*(uri = "auto", params: openarray[string] = []): Store = mitNix: @@ -48,13 +48,13 @@ proc getUri*(store: Store; cb: StringCallback) = mitNix: let state = new StringCallbackState state.callback = cb - discard nix.store_get_uri(store, receiveString, state[].addr) + checkError nix.store_get_uri(store, receiveString, state[].addr) proc getVersion*(store: Store; cb: StringCallback) = mitNix: let state = new StringCallbackState state.callback = cb - discard nix.store_get_version(store, receiveString, state[].addr) + checkError nix.store_get_version(store, receiveString, state[].addr) proc isValidPath*(store: Store; path: string): bool = assert not store.isNil @@ -74,9 +74,9 @@ proc copyClosure*(src, dst: Store; path: string) = if sp.isNil: raise newException(CatchableError, "store_parse_path failed") defer: store_path_free(sp) - nix.store_copy_closure(src, dst, sp) + checkError nix.store_copy_closure(src, dst, sp) -proc newState*(store: Store; lookupPath: openarray[string]): EvalState = +proc newState*(store: Store; lookupPath: openarray[string] = []): EvalState = mitNix: var path = allocCStringArray(lookupPath) defer: deallocCStringArray(path) @@ -87,23 +87,19 @@ proc close*(state: EvalState) = state_free(state) proc close*(value: Value) = mitNix: - discard nix.gc_decref(cast[pointer](value)) + checkError nix.gc_decref(cast[pointer](value)) proc evalFromString*(nix: NixContext; state: EvalState; expr, path: string; result: Value) = - discard nix.expr_eval_from_string(state, expr, path, result) + checkError nix.expr_eval_from_string(state, expr, path, result) -proc evalFromString*(state: EvalState; expr: string; path = ""): Value = +proc evalFromString*(state: EvalState; expr: string; path = "."): Value = mitNix: - try: - result = nix.alloc_value(state) - nix.evalFromString(state, expr, path, result) - except CatchableError as err: - result.close() - raise err + result = nix.alloc_value(state) + nix.evalFromString(state, expr, path, result) proc force*(state: EvalState; value: Value) = mitNix: - discard nix.value_force(state, value) + checkError nix.value_force(state, value) proc get_attr_byidx*(value: Value; state: EvalState; i: cuint): (cstring, Value) = mitNix: @@ -111,7 +107,7 @@ proc get_attr_byidx*(value: Value; state: EvalState; i: cuint): (cstring, Value) proc apply(nix: NixContext; state: EvalState; fn, arg: Value): Value = result = nix.alloc_value(state) - discard nix.init_apply(result, fn, arg) + checkError nix.init_apply(result, fn, arg) proc apply*(state: EvalState; fn, arg: Value): Value = mitNix: @@ -121,4 +117,4 @@ proc call*(state: EvalState; fn: Value; args: varargs[Value]): Value = mitNix: result = nix.alloc_value(state) var array = cast[ptr UncheckedArray[Value]](args) - discard nix.value_call_multi(state, fn, args.len.csize_t, array, result) + checkError nix.value_call_multi(state, fn, args.len.csize_t, array, result) diff --git a/src/nix_actor/nix_api_store.nim b/src/nix_actor/nix_api_store.nim index 0e1719062248..2df3f891ede4 100644 --- a/src/nix_actor/nix_api_store.nim +++ b/src/nix_actor/nix_api_store.nim @@ -7,17 +7,17 @@ import ./nix_api_types {.pragma: nix_api_store, header: "nix_api_store.h", importc: "nix_$1".} -proc libstore_init*(context: NixContext): nix_err {.nix_api_store, discardable.} +proc libstore_init*(context: NixContext): nix_err {.nix_api_store.} -proc libstore_init_no_load_config*(context: NixContext): nix_err {.nix_api_store, discardable.} +proc libstore_init_no_load_config*(context: NixContext): nix_err {.nix_api_store.} -proc init_plugins*(context: NixContext): nix_err {.nix_api_store, discardable.} +proc init_plugins*(context: NixContext): nix_err {.nix_api_store.} proc store_open*(context: NixContext; uri: cstring; params: ptr cstringArray): Store {.nix_api_store.} proc store_free*(store: Store) {.nix_api_store.} -proc store_get_uri*(context: NixContext; store: Store; callback: GetStringCallback; user_data: pointer): nix_err {.nix_api_store, discardable.} +proc store_get_uri*(context: NixContext; store: Store; callback: GetStringCallback; user_data: pointer): nix_err {.nix_api_store.} proc store_parse_path*(context: NixContext; store: Store; path: cstring): StorePath {.nix_api_store.} @@ -31,8 +31,8 @@ proc store_is_valid_path*(context: NixContext; store: Store; path: StorePath): b type RealiseCallback* = proc (userdata: pointer; outname: cstring; `out`: cstring) {.cdecl.} -proc store_realise*(context: NixContext; store: Store; path: StorePath; userdata: pointer; callback: RealiseCallback): nix_err {.nix_api_store, discardable.} +proc store_realise*(context: NixContext; store: Store; path: StorePath; userdata: pointer; callback: RealiseCallback): nix_err {.nix_api_store.} -proc store_get_version*(context: NixContext; store: Store; callback: GetStringCallback; user_data: pointer): nix_err {.nix_api_store, discardable.} +proc store_get_version*(context: NixContext; store: Store; callback: GetStringCallback; user_data: pointer): nix_err {.nix_api_store.} -proc store_copy_closure*(context: NixContext; src, dst: Store; path: StorePath): nix_err {.nix_api_store, discardable.} +proc store_copy_closure*(context: NixContext; src, dst: Store; path: StorePath): nix_err {.nix_api_store.} diff --git a/src/nix_actor/nix_api_value.nim b/src/nix_actor/nix_api_value.nim index 5fba2af56a58..d4d5edc5f4c3 100644 --- a/src/nix_actor/nix_api_value.nim +++ b/src/nix_actor/nix_api_value.nim @@ -18,7 +18,7 @@ proc get_typename*(context: NixContext; value: Value): cstring {.nix_api_value.} proc get_bool*(context: NixContext; value: Value): bool {.nix_api_value.} -proc get_string*(context: NixContext; value: Value; callback: GetStringCallback; user_data: pointer): nix_err {.nix_api_value, discardable.} +proc get_string*(context: NixContext; value: Value; callback: GetStringCallback; user_data: pointer): nix_err {.nix_api_value.} proc get_path_string*(context: NixContext; value: Value): cstring {.nix_api_value.} @@ -42,39 +42,39 @@ proc get_attr_byidx*(context: NixContext; value: Value; state: EvalState; i: cui # proc get_attr_name_byidx*(context: NixContext; value: Value; state: EvalState; i: cuint): cstring {.nix_api_value.} -proc init_bool*(context: NixContext; value: Value; b: bool): nix_err {.nix_api_value, discardable.} +proc init_bool*(context: NixContext; value: Value; b: bool): nix_err {.nix_api_value.} -proc init_string*(context: NixContext; value: Value; str: cstring): nix_err {.nix_api_value, discardable.} +proc init_string*(context: NixContext; value: Value; str: cstring): nix_err {.nix_api_value.} -proc init_path_string*(context: NixContext; s: EvalState; value: Value; str: cstring): nix_err {.nix_api_value, discardable.} +proc init_path_string*(context: NixContext; s: EvalState; value: Value; str: cstring): nix_err {.nix_api_value.} -proc init_float*(context: NixContext; value: Value; d: cdouble): nix_err {.nix_api_value, discardable.} +proc init_float*(context: NixContext; value: Value; d: cdouble): nix_err {.nix_api_value.} -proc init_int*(context: NixContext; value: Value; i: int64): nix_err {.nix_api_value, discardable.} +proc init_int*(context: NixContext; value: Value; i: int64): nix_err {.nix_api_value.} -proc init_null*(context: NixContext; value: Value): nix_err {.nix_api_value, discardable.} +proc init_null*(context: NixContext; value: Value): nix_err {.nix_api_value.} -proc init_apply*(context: NixContext; value: Value; fn: Value; arg: Value): nix_err {.nix_api_value, discardable.} +proc init_apply*(context: NixContext; value: Value; fn: Value; arg: Value): nix_err {.nix_api_value.} -proc init_external*(context: NixContext; value: Value; val: ExternalValue): nix_err {.nix_api_value, discardable.} +proc init_external*(context: NixContext; value: Value; val: ExternalValue): nix_err {.nix_api_value.} -proc make_list*(context: NixContext; list_builder: ListBuilder; value: Value): nix_err {.nix_api_value, discardable.} +proc make_list*(context: NixContext; list_builder: ListBuilder; value: Value): nix_err {.nix_api_value.} proc make_list_builder*(context: NixContext; state: EvalState; capacity: csize_t): ListBuilder {.nix_api_value.} -proc list_builder_insert*(context: NixContext; list_builder: ListBuilder; index: cuint; value: Value): nix_err {.nix_api_value, discardable.} +proc list_builder_insert*(context: NixContext; list_builder: ListBuilder; index: cuint; value: Value): nix_err {.nix_api_value.} proc list_builder_free*(list_builder: ListBuilder) {.nix_api_value.} -proc make_attrs*(context: NixContext; value: Value; b: BindingsBuilder): nix_err {.nix_api_value, discardable.} +proc make_attrs*(context: NixContext; value: Value; b: BindingsBuilder): nix_err {.nix_api_value.} -# proc init_primop*(context: NixContext; value: Value; op: PrimOp): nix_err {.nix_api_value, discardable.} +# proc init_primop*(context: NixContext; value: Value; op: PrimOp): nix_err {.nix_api_value.} -proc copy_value*(context: NixContext; value: Value; source: Value): nix_err {.nix_api_value, discardable.} +proc copy_value*(context: NixContext; value: Value; source: Value): nix_err {.nix_api_value.} proc make_bindings_builder*(context: NixContext; state: EvalState; capacity: csize_t): BindingsBuilder {.nix_api_value.} -proc bindings_builder_insert*(context: NixContext; builder: BindingsBuilder; name: cstring; value: Value): nix_err {.nix_api_value, discardable.} +proc bindings_builder_insert*(context: NixContext; builder: BindingsBuilder; name: cstring; value: Value): nix_err {.nix_api_value.} proc bindings_builder_free*(builder: BindingsBuilder) {.nix_api_value.} diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim index e97d19974daa..83b896be7595 100644 --- a/src/nix_actor/nix_values.nim +++ b/src/nix_actor/nix_values.nim @@ -51,27 +51,26 @@ proc exportNix*(facet: Facet; v: Value): Value = v v.mapEmbeds(op) -proc callThru(state: EvalState; nv: NixValue): NixValue = +proc callThru(nix: NixContext; state: EvalState; nv: NixValue): NixValue = result = nv - mitNix: - while true: - case nix.get_type(result) - of NIX_TYPE_THUNK: - state.force(result) - of NIX_TYPE_FUNCTION: - # Call functions with empty attrsets. - var - args = nix.alloc_value(state) - bb = nix.make_bindings_builder(state, 0) - discard nix.gc_decref(args) - doAssert nix.make_attrs(args, bb) == NIX_OK - bindings_builder_free(bb) - result = state.apply(result, args) - else: - return + while true: + case nix.get_type(result) + of NIX_TYPE_THUNK: + state.force(result) + of NIX_TYPE_FUNCTION: + # Call functions with empty attrsets. + var + args = nix.alloc_value(state) + bb = nix.make_bindings_builder(state, 0) + checkError nix.gc_decref(args) + doAssert nix.make_attrs(args, bb) == NIX_OK + bindings_builder_free(bb) + result = state.apply(result, args) + else: + return -proc toPreserves*(state: EvalState; value: NixValue; nix: NixContext): Value {.gcsafe.} = - var value = callThru(state, value) +proc toPreserves*(value: NixValue; state: EvalState; nix: NixContext): Value {.gcsafe.} = + var value = nix.callThru(state, value) let kind = nix.get_type(value) case kind @@ -95,11 +94,11 @@ proc toPreserves*(state: EvalState; value: NixValue; nix: NixContext): Value {.g var str = nix.get_attr_byname(value, state, "__toString") if nix.get_type(str) == NIX_TYPE_FUNCTION: str = state.apply(str, value) - result = state.toPreserves(str, nix) + result = str.toPreserves(state, nix) elif nix.has_attr_byname(value, state, "outPath"): var outPath = nix.get_attr_byname(value, state, "outPath") result = Derivation( - value: state.toPreserves(outPath, nix), + value: outPath.toPreserves(state, nix), context: NixValueRef(value: value).embed, ).toPreserves else: @@ -108,7 +107,7 @@ proc toPreserves*(state: EvalState; value: NixValue; nix: NixContext): Value {.g var i: cuint while i < n: let (key, val) = get_attr_byidx(value, state, i) - result[($key).toSymbol] = state.toPreserves(val, nix) + result[($key).toSymbol] = val.toPreserves(state, nix) inc(i) of NIX_TYPE_LIST: let n = nix.getListSize(value) @@ -116,37 +115,37 @@ proc toPreserves*(state: EvalState; value: NixValue; nix: NixContext): Value {.g var i: cuint while i < n: var val = nix.getListByIdx(value, state, i) - result[i] = state.toPreserves(val, nix) + result[i] = val.toPreserves(state, nix) inc(i) of NIX_TYPE_THUNK, NIX_TYPE_FUNCTION: raiseAssert "cannot preserve thunk or function" of NIX_TYPE_EXTERNAL: result = "«external»".toPreserves -proc toPreserves*(state: EvalState; value: NixValue): Value {.gcsafe.} = - mitNix: result = toPreserves(state, value, nix) +proc toPreserves*(value: NixValue; state: EvalState): Value {.gcsafe.} = + mitNix: result = toPreserves(value, state, nix) proc translate*(nix: NixContext; state: EvalState; pr: preserves.Value): NixValue = try: result = nix.alloc_value(state) case pr.kind of pkBoolean: - nix.init_bool(result, pr.bool) + checkError nix.init_bool(result, pr.bool) of pkFloat: - nix.init_float(result, pr.float.cdouble) + checkError nix.init_float(result, pr.float.cdouble) of pkRegister: - nix.init_int(result, pr.register.int64) + checkError nix.init_int(result, pr.register.int64) of pkBigInt: - nix.init_int(result, pr.register.int64) + checkError nix.init_int(result, pr.register.int64) of pkString: - nix.init_string(result, pr.string) + checkError nix.init_string(result, pr.string) of pkByteString: raise newException(ValueError, "cannot convert large Preserves integer to Nix: " & $pr) of pkSymbol: nix.evalFromString(state, cast[string](pr.symbol), "", result) of pkRecord: if pr.isRecord("null", 0): - nix.init_null(result) + checkError nix.init_null(result) elif pr.isRecord("drv", 2): var drv: Derivation if not drv.fromPreserves(pr): @@ -161,17 +160,17 @@ proc translate*(nix: NixContext; state: EvalState; pr: preserves.Value): NixValu let b = nix.make_list_builder(state, pr.len.csize_t) defer: list_builder_free(b) for i, e in pr: - discard nix.list_builder_insert(b, i.register.cuint, nix.translate(state, e)) - nix.make_list(b, result) + checkError nix.list_builder_insert(b, i.register.cuint, nix.translate(state, e)) + checkError nix.make_list(b, result) of pkDictionary: let b = nix.make_bindings_builder(state, pr.dict.len.csize_t) defer: bindings_builder_free(b) for (name, value) in pr.dict: if name.isSymbol: - nix.bindings_builder_insert(b, name.symbol.string, nix.translate(state, value)) + checkError nix.bindings_builder_insert(b, name.symbol.string, nix.translate(state, value)) else: - nix.bindings_builder_insert(b, $name, nix.translate(state, value)) - nix.make_attrs(result, b) + checkError nix.bindings_builder_insert(b, $name, nix.translate(state, value)) + checkError nix.make_attrs(result, b) of pkEmbedded: raise newException(ValueError, "cannot convert Preserves embedded value to Nix") except CatchableError as err: @@ -183,9 +182,9 @@ proc toNix*(pr: preserves.Value; state: EvalState): NixValue = result = nix.translate(state, pr) proc step*(state: EvalState; nv: NixValue; path: openarray[preserves.Value]): Option[preserves.Value] = - var nv = callThru(state, nv) mitNix: var + nv = nix.callThru(state, nv) i = 0 while i < path.len: if nv.isNil: return @@ -223,3 +222,17 @@ proc realise*(nix: NixContext; state: EvalState; val: NixValue): Value = proc realise*(state: EvalState; val: NixValue): Value = mitNix: result = nix.realise(state, val) + +proc initNull*(state: EvalState): NixValue = + mitNix: + result = nix.alloc_value(state) + checkError nix.init_null(result) + +proc isFunc*(value: NixValue): bool = + mitNix: result = nix.get_type(value) == NIX_TYPE_FUNCTION + +proc isNull*(value: NixValue): bool = + mitNix: result = nix.get_type(value) == NIX_TYPE_NULL + +proc isThunk*(value: NixValue): bool = + mitNix: result = nix.get_type(value) == NIX_TYPE_THUNK diff --git a/src/nix_actor/utils.nim b/src/nix_actor/utils.nim index fd28ba1e7b6a..84b3e0ea65fb 100644 --- a/src/nix_actor/utils.nim +++ b/src/nix_actor/utils.nim @@ -14,11 +14,13 @@ proc newException(ctx: NixContext): ref NixException = if n > 0: copyMem(result.msg[0].addr, p, result.msg.len) +template checkError*(code: nix_err) = + if code != NIX_OK: raise newException(nix) + template mitNix*(body: untyped): untyped = ## Mit nix machen. block: var nix {.inject.} = c_context_create() defer: c_context_free(nix) body - if err_code(nix) != NIX_OK: - let err = newException(nix) + checkError err_code(nix) -- cgit 1.4.1 From 151037cf4dac74a3d9eb2cbe84f167a21c50d820 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sun, 25 Aug 2024 20:42:23 +0300 Subject: Replace with and MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- protocol.prs | 52 +++--- src/nix_actor.nim | 224 ++++++++++++-------------- src/nix_actor/nix_api.nim | 6 + src/nix_actor/nix_values.nim | 37 ++++- src/nix_actor/old.nim | 369 ------------------------------------------- src/nix_actor/protocol.nim | 151 +++++++++--------- test.pr | 1 + tests/test.nim | 111 +++++++++---- 8 files changed, 311 insertions(+), 640 deletions(-) delete mode 100644 src/nix_actor/old.nim diff --git a/protocol.prs b/protocol.prs index 2852ff103cc9..c39a44a6853d 100644 --- a/protocol.prs +++ b/protocol.prs @@ -1,43 +1,32 @@ version 1 . embeddedType EntityRef.Cap . -# Gatekeeper step to access a Nix repository. -RepoResolveStep = . -RepoResolveDetail = { - - # Path to a repository to import. - # This string is evaluated so it can include "<…>" paths. - import: string - - # List of strings corresponding to entries in NIX_PATH. - # For example: - # [ "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos" - # "nixos-config=/etc/nixos/configuration.nix" - # "/nix/var/nix/profiles/per-user/root/channels" - # ] - lookupPath: [string ...] - -} & @args RepoArgs & @store RepoStore & @cacheSpace CacheSpace . +# Gatekeeper step to access a Nix store. +NixResolveStep = . -# Arguments to call the imported expression with if it is a function. -RepoArgs = @present { args: any } / @absent { } . +NixResolveDetail = { } +& @lookupPath LookupPath +& @storeUri StoreUri +& @storeParams StoreParams +& @cacheSpace CacheSpace +. -# Store uri or capability. -RepoStore = @uri { store: string } / @cap { store: #:any } / @absent { } . +# List of strings corresponding to entries in NIX_PATH. +# For example: +# [ "nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixos" +# "nixos-config=/etc/nixos/configuration.nix" +# "/nix/var/nix/profiles/per-user/root/channels" +# ] +LookupPath = @lookupPath { lookupPath: [string ...] } / @absent { } . -# Common error type. -Error = . +StoreUri = @storeUri { storeUri: string } / @absent { } . -# Gatekeeper step to access a Nix store. -StoreResolveStep = . -StoreResolveDetail = { - params: AttrSet - uri: string -} & @cacheSpace CacheSpace . +StoreParams = @storeParams { storeParams: AttrSet } / @absent { } . # A cooperative caching dataspace shared with other Nix stores. CacheSpace = @cacheSpace { cache: #:any } / @absent { } . +# Assertion. CheckStorePath = . # Represents a Nix derivation. The @value can be realised via @context. @@ -46,8 +35,6 @@ Derivation = . # Assertion. Realise = . -Context = #:any . - # Assertion. The store that this asserted to will copy the closure of @storePath to @destination. When the copy completes or fails a Result value is asserted to @result. CopyClosure = . @@ -61,3 +48,6 @@ AttrSet = {symbol: any ...:...} . # Value. Result = Error / . + +# Common error type. +Error = . diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 4e4bde699541..e19b1397e251 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -7,7 +7,7 @@ import pkg/preserves/sugar, pkg/syndicate, pkg/syndicate/[gatekeepers, patterns, relays], - ./nix_actor/[nix_api, nix_values], + ./nix_actor/[nix_api, nix_api_value, nix_values, utils], ./nix_actor/protocol proc echo(args: varargs[string, `$`]) {.used.} = @@ -34,124 +34,107 @@ proc unembedEntity(v: Value; E: typedesc): Option[E] = if v.isEmbeddedRef: result = v.embeddedRef.unembedEntity(E) +proc openStore(uri: string; params: Option[AttrSet]): Store = + var pairs: seq[string] + if params.isSome: + var i: int + pairs.setLen(params.get.len) + for (key, val) in params.get.pairs: + pairs[i] = $key & "=" & $val + inc i + openStore(uri, pairs) + type - StoreEntity {.final.} = ref object of Entity - self: Cap + NixState = object store: Store + eval: EvalState -proc openStore(uri: string; params: AttrSet): Store = - var - pairs = newSeq[string](params.len) - i: int - for (key, val) in params.pairs: - pairs[i] = $key & "=" & $val - inc i - openStore(uri, pairs) +proc initState(detail: NixResolveDetail): NixState = + result.store = openStore(detail.storeUri.get("auto"), detail.storeParams) + if detail.lookupPath.isSome: + result.eval = newState(result.store, detail.lookupPath.get) + else: + result.eval = newState(result.store) + +proc close(state: NixState) = + if not state.eval.isNil: + state.eval.close() + if not state.store.isNil: + state.store.close() + +type + NixEntity {.acyclic, final.} = ref object of Entity + state: NixState + self: Cap + root: NixValue -proc newStoreEntity(turn: Turn; detail: StoreResolveDetail): StoreEntity = - let entity = StoreEntity() +proc newNixEntity(turn: Turn; detail: NixResolveDetail): NixEntity = + let entity = NixEntity(state: initState(detail)) + entity.root = entity.state.eval.initNull() turn.onStop do (turn: Turn): - if not entity.store.isNil: - entity.store.close() - reset entity.store - entity.store = openStore(detail.uri, detail.params) + decref(entity.root) + entity.state.close() entity.self = newCap(turn, entity) entity -proc serve(entity: StoreEntity; turn: Turn; checkPath: CheckStorePath) = +proc newChild(parent: NixEntity; turn: Turn; val: NixValue): NixEntity = + ## Create a child entity for a given root value. + let entity = NixEntity(state: parent.state, root: val) + entity.state.eval.force(entity.root) + entity.self = newCap(turn, entity) + turn.onStop do (turn: Turn): + decref(entity.root) + entity + +proc serve(entity: NixEntity; turn: Turn; checkPath: CheckStorePath) = tryPublish(turn, checkPath.valid.Cap): - let v = entity.store.isValidPath(checkPath.path) + let v = entity.state.store.isValidPath(checkPath.path) publish(turn, checkPath.valid.Cap, initRecord("ok", %v)) -proc serve(entity: StoreEntity; turn: Turn; obs: Observe) = - let facet = turn.facet - if obs.pattern.matches(initRecord("uri", %"")): - entity.store.getUri do (s: string): - facet.run do (turn: Turn): - publish(turn, obs.observer.Cap, obs.pattern.capture(initRecord("uri", %s)).get) - if obs.pattern.matches(initRecord("version", %"")): - entity.store.getVersion do (s: string): - facet.run do (turn: Turn): - publish(turn, obs.observer.Cap, obs.pattern.capture(initRecord("version", %s)).get) - -proc serve(entity: StoreEntity; turn: Turn; copy: CopyClosure) = - var dest = copy.dest.unembedEntity(StoreEntity) +proc serve(entity: NixEntity; turn: Turn; copy: CopyClosure) = + var dest = copy.dest.unembedEntity(NixEntity) if dest.isNone: publishError(turn, copy.result.Cap, %"destination store is not colocated with source store") else: tryPublish(turn, copy.result.Cap): - entity.store.copyClosure(dest.get.store, copy.storePath) + entity.state.store.copyClosure(dest.get.state.store, copy.storePath) publishOk(turn, copy.result.Cap, %true) # TODO: assert some stats or something. -method publish(entity: StoreEntity; turn: Turn; a: AssertionRef; h: Handle) = - var - # orc doesn't handle this as a union object - observe: Observe - checkPath: CheckStorePath - copyClosure: CopyClosure - if checkPath.fromPreserves(a.value): - entity.serve(turn, checkPath) - elif observe.fromPreserves(a.value): - entity.serve(turn, observe) - elif copyClosure.fromPreserves(a.value) and - copyClosure.result of Cap: - entity.serve(turn, copyClosure) - else: - when not defined(release): - echo "nix-store: unhandled assertion ", a.value - -type - RepoEntity {.final.} = ref object of Entity - self: Cap - store: StoreEntity - state: EvalState - root: NixValue - -proc newRepoEntity(turn: Turn; detail: RepoResolveDetail): RepoEntity = - let entity = RepoEntity() - turn.onStop do (turn: Turn): - if not entity.state.isNil: - entity.state.close() - if detail.store.isSome: - var other = detail.store.get.unembedEntity(StoreEntity) - if other.isSome: - entity.store = other.get - elif detail.store.get.isString: - var storeDetail = StoreResolveDetail(cache: detail.cache, uri: detail.store.get.string) - entity.store = newStoreEntity(turn, storeDetail) - else: - raise newException(CatchableError, "invalid store parameter for nix-repo: " & $detail.store.get) +proc serve(entity: NixEntity; turn: Turn; obs: Observe) = + let facet = turn.facet + #[ + # TODO: move to a store entity + if obs.pattern.matches(initRecord("uri", %"")): + entity.state.store.getUri do (s: string): + facet.run do (turn: Turn): + publish(turn, obs.observer.Cap, obs.pattern.capture(initRecord("uri", %s)).get) + elif obs.pattern.matches(initRecord("version", %"")): + entity.state.store.getVersion do (s: string): + facet.run do (turn: Turn): + publish(turn, obs.observer.Cap, obs.pattern.capture(initRecord("version", %s)).get) else: - var storeDetail = StoreResolveDetail(cache: detail.cache, uri: "auto") - entity.store = newStoreEntity(turn, storeDetail) - entity.state = newState(entity.store.store, detail.lookupPath) - entity.root = entity.state.evalFromString("import " & detail.`import`) - if detail.args.isSome: - var na = detail.args.get.toNix(entity.state) - entity.root = entity.state.apply(entity.root, na) - entity.self = newCap(turn, entity) - entity - -proc serve(repo: RepoEntity; turn: Turn; obs: Observe) = + ]# var analysis = analyse(obs.pattern) captures = newSeq[Value](analysis.capturePaths.len) block stepping: for i, path in analysis.constPaths: - var v = repo.state.step(repo.root, path) + var v = entity.state.eval.step(entity.root, path) if v.isNone or v.get != analysis.constValues[i]: let null = initRecord("null") for v in captures.mitems: v = null break stepping for i, path in analysis.capturePaths: - var v = repo.state.step(repo.root, path) + var v = entity.state.eval.step(entity.root, path) + assert v.isSome if v.isSome: captures[i] = turn.facet.exportNix(v.get) - else: captures[i] = initRecord("null") - discard publish(turn, Cap obs.observer, captures) + else: + captures[i] = initRecord("null") + publish(turn, Cap obs.observer, captures) -proc serve(repo: RepoEntity; turn: Turn; r: Realise) = +proc serve(entity: NixEntity; turn: Turn; r: Realise) = tryPublish(turn, r.result.Cap): var drv: Derivation if not drv.fromPreserves(r.value): @@ -164,50 +147,51 @@ proc serve(repo: RepoEntity; turn: Turn; r: Realise) = if not(dummyCap.get.target of NixValueRef): publishError(turn, r.result.Cap, %"derivation context is not a NixValueRef") else: - let v = repo.state.realise(dummyCap.get.target.NixValueRef.value) + let v = entity.state.eval.realise(dummyCap.get.target.NixValueRef.value) publishOk(turn, r.result.Cap, v) # TODO: this is awkward. -proc serve(repo: RepoEntity; turn: Turn; eval: Eval) = - tryPublish(turn, eval.result.Cap): - var - expr = repo.state.evalFromString(eval.expr) - args = eval.args.toNix(repo.state) - val = repo.state.call(expr, repo.root, args) - res = repo.state.toPreserves(val) - publishOk(turn, eval.result.Cap, res) - -method publish(repo: RepoEntity; turn: Turn; a: AssertionRef; h: Handle) = - ## Respond to observations with dataspace semantics, minus retraction - ## of assertions in response to the retraction of observations. - ## This entity is scoped to immutable data so this shouldn't be a problem. +proc serve(entity: NixEntity; turn: Turn; e: Eval) = + tryPublish(turn, e.result.Cap): + var expr = entity.state.eval.evalFromString(e.expr) + expr = entity.state.eval.apply(expr, entity.root) + expr = entity.state.eval.apply(expr, e.args.toNix(entity.state.eval)) + publishOk(turn, e.result.Cap, entity.newChild(turn, expr).self.toPreserves) + +method publish(entity: NixEntity; turn: Turn; a: AssertionRef; h: Handle) = var - obs: Observe - realise: Realise + # orc doesn't handle this as a union object + checkPath: CheckStorePath + copyClosure: CopyClosure eval: Eval - if obs.fromPreserves(a.value) and obs.observer of Cap: - serve(repo, turn, obs) + observe: Observe + realise: Realise + + if checkPath.fromPreserves(a.value): + entity.serve(turn, checkPath) + elif observe.fromPreserves(a.value): + entity.serve(turn, observe) + elif copyClosure.fromPreserves(a.value) and + copyClosure.result of Cap: + entity.serve(turn, copyClosure) + elif observe.fromPreserves(a.value) and observe.observer of Cap: + serve(entity, turn, observe) elif realise.fromPreserves(a.value) and realise.result of Cap: - serve(repo, turn, realise) + serve(entity, turn, realise) elif eval.fromPreserves(a.value) and eval.result of Cap: - serve(repo, turn, eval) + serve(entity, turn, eval) else: when not defined(release): - echo "nix-repo: unhandled assertion ", a.value + echo "unhandled assertion ", a.value -proc main() = +proc bootActor*(turn: Turn; relay: Cap) = initLibstore() initLibexpr() - runActor("main") do (turn: Turn): - resolveEnvironment(turn) do (turn: Turn; relay: Cap): - let gk = spawnGatekeeper(turn, relay) - - gk.serve do (turn: Turn; step: StoreResolveStep) -> Resolved: - newStoreEntity(turn, step.detail).self.resolveAccepted + let gk = spawnGatekeeper(turn, relay) + gk.serve do (turn: Turn; step: NixResolveStep) -> Resolved: + newNixEntity(turn, step.detail).self.resolveAccepted - gk.serve do (turn: Turn; step: RepoResolveStep) -> Resolved: - newRepoEntity(turn, step.detail).self.resolveAccepted - # TODO: support for stepping through attrs? - -main() +when isMainModule: + runActor("main") do (turn: Turn): + resolveEnvironment(turn, bootActor) diff --git a/src/nix_actor/nix_api.nim b/src/nix_actor/nix_api.nim index e382602ca8c1..6550e589b2b7 100644 --- a/src/nix_actor/nix_api.nim +++ b/src/nix_actor/nix_api.nim @@ -118,3 +118,9 @@ proc call*(state: EvalState; fn: Value; args: varargs[Value]): Value = result = nix.alloc_value(state) var array = cast[ptr UncheckedArray[Value]](args) checkError nix.value_call_multi(state, fn, args.len.csize_t, array, result) + +proc incref*(v: Value) = + mitNix: checkError nix.gc_incref(cast[pointer](v)) + +proc decref*(v: Value) = + mitNix: checkError nix.gc_decref(cast[pointer](v)) diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim index 83b896be7595..8c87f51a9a97 100644 --- a/src/nix_actor/nix_values.nim +++ b/src/nix_actor/nix_values.nim @@ -32,7 +32,7 @@ proc unthunk(v: Value): Value = thunk.get.data.get.toPreserves else: v -proc unthunkAll(v: Value): Value = +proc unthunkAll*(v: Value): Value = v.mapEmbeds(unthunk) proc exportNix*(facet: Facet; v: Value): Value = @@ -210,8 +210,8 @@ proc step*(state: EvalState; nv: NixValue; path: openarray[preserves.Value]): Op inc i else: raiseAssert("cannot step " & $kind) - return - result = state.toPreserves(nv).some + result = nv.toPreserves(state, nix).some + assert path.len > 0 or result.isSome proc realise*(nix: NixContext; state: EvalState; val: NixValue): Value = result = "".toPreserves @@ -228,11 +228,34 @@ proc initNull*(state: EvalState): NixValue = result = nix.alloc_value(state) checkError nix.init_null(result) -proc isFunc*(value: NixValue): bool = - mitNix: result = nix.get_type(value) == NIX_TYPE_FUNCTION +proc typeName*(val: NixValue): string = + mitNix: + result = $nix.get_type(val) + # result = $nix.get_typename(val) + +proc isThunk*(value: NixValue): bool = + mitNix: result = nix.get_type(value) == NIX_TYPE_THUNK + +proc isLiteral*(value: NixValue): bool = + mitNix: + let kind = nix.get_type(value) + result = + case kind + of NIX_TYPE_INT, + NIX_TYPE_FLOAT, + NIX_TYPE_BOOL, + NIX_TYPE_STRING, + NIX_TYPE_PATH, + NIX_TYPE_NULL, + NIX_TYPE_ATTRS, + NIX_TYPE_LIST: + true + of NIX_TYPE_THUNK, NIX_TYPE_FUNCTION, + NIX_TYPE_EXTERNAL: + false proc isNull*(value: NixValue): bool = mitNix: result = nix.get_type(value) == NIX_TYPE_NULL -proc isThunk*(value: NixValue): bool = - mitNix: result = nix.get_type(value) == NIX_TYPE_THUNK +proc isFunc*(value: NixValue): bool = + mitNix: result = nix.get_type(value) == NIX_TYPE_FUNCTION diff --git a/src/nix_actor/old.nim b/src/nix_actor/old.nim deleted file mode 100644 index 78ef76237be9..000000000000 --- a/src/nix_actor/old.nim +++ /dev/null @@ -1,369 +0,0 @@ -type - Snoop = ref object - client, daemon: AsyncSocket - buffer: seq[Word] - version: Version - -type ValidPathInfo = object - path: string - deriver: string - narHash: string - references: StringSet - registrationTime, narSize: BiggestInt - ultimate: bool - sigs: StringSet - ca: string - -proc send(session: Snoop; sock: AsyncSocket; words: varargs[Word]): Future[void] = - for i, word in words: session.buffer[i] = word - send(sock, addr session.buffer[0], words.len shl 3) - -proc send(session: Snoop; sock: AsyncSocket; s: string): Future[void] = - let wordCount = (s.len + 7) shr 3 - if wordCount > session.buffer.len: setLen(session.buffer, wordCount) - session.buffer[0] = Word s.len - if wordCount > 0: - session.buffer[wordCount] = 0x00 - copyMem(addr session.buffer[1], unsafeAddr s[0], s.len) - send(sock, addr session.buffer[0], (succ wordCount) shl 3) - -proc passWord(a, b: AsyncSocket): Future[Word] {.async.} = - var w = await recvWord(a) - await send(b, addr w, sizeof(Word)) - return w - -proc passString(session: Snoop; a, b: AsyncSocket): Future[string] {.async.} = - var s = await recvString(a) - await send(session, b, s) - return s - -proc passStringSeq(session: Snoop; a, b: AsyncSocket): Future[seq[string]] {.async.} = - let count = int(await passWord(a, b)) - var strings = newSeq[string](count) - for i in 0..= 16 - info.ultimate = (await passDaemonWord(session)) != 0 - info.sigs = await passDaemonStringSet(session) - info.ca = await passDaemonString(session) - return info - -proc passChunks(session: Snoop; a, b: AsyncSocket): Future[int] {.async.} = - var total: int - while true: - let chunkLen = int(await passWord(a, b)) - if chunkLen == 0: - break - else: - let wordLen = (chunkLen + 7) shr 3 - if session.buffer.len < wordLen: setLen(session.buffer, wordLen) - let recvLen = await recvInto(a, addr session.buffer[0], chunkLen) - # each chunk must be recved contiguously - if recvLen != chunkLen: - raise newException(ProtocolError, "invalid chunk read") - await send(b, addr session.buffer[0], recvLen) - inc(total, recvLen) - return total - -proc passClientChunks(session: Snoop): Future[int] = - passChunks(session, session.client, session.daemon) - -proc passErrorDaemonError(session: Snoop) {.async.} = - let - typ = await passDaemonString(session) - assert typ == "Error" - let - lvl = await passDaemonWord(session) - name = await passDaemonString(session) - msg = passDaemonString(session) - havePos = await passDaemonWord(session) - assert havePos == 0 - let - nrTraces = await passDaemonWord(session) - for i in 1..nrTraces: - let havPos = await passDaemonWord(session) - assert havPos == 0 - let msg = await passDaemonString(session) - -proc passDaemonFields(session: Snoop): Future[Fields] {.async.} = - let count = await passDaemonWord(session) - var fields = newSeq[Field](count) - for i in 0..= 26 - await passErrorDaemonError(session) - - of STDERR_NEXT: - let s = await passDaemonString(session) - - of STDERR_START_ACTIVITY: - var act: ActionStart - act.id = BiggestInt(await passDaemonWord(session)) - act.level = BiggestInt(await passDaemonWord(session)) - act.`type` = BiggestInt(await passDaemonWord(session)) - act.text = await passDaemonString(session) - act.fields = await passDaemonFields(session) - act.parent = BiggestInt(await passDaemonWord(session)) - - of STDERR_STOP_ACTIVITY: - var act: ActionStop - act.id = BiggestInt(await passDaemonWord(session)) - - of STDERR_RESULT: - var act: ActionResult - act.id = BiggestInt(await passDaemonWord(session)) - act.`type` = BiggestInt(await passDaemonWord(session)) - act.fields = await passDaemonFields(session) - - of STDERR_LAST: - break - - else: - raise newException(ProtocolError, "unknown work verb " & $word) - -#[ -proc fromClient(miss: var Missing; socket: AsyncSocket) {.async.} = - result.targets = await passClientStringSet(session) - -proc fromDaemon(miss: var Missing; socket: AsyncSocket) {.async.} = - miss.willBuild = await passDaemonStringSet(session) - miss.willSubstitute = await passDaemonStringSet(session) - miss.unknown = await passDaemonStringSet(session) - miss.downloadSize = BiggestInt await passDaemonWord(session) - miss.narSize = BiggestInt await passDaemonWord(session) -]# - -proc loop(session: Snoop) {.async.} = - var chunksTotal: int - try: - while not session.client.isClosed: - let wop = await passClientWord(session) - case wop - of wopIsValidPath: - let path = await passClientString(session) - stderr.writeLine "wopIsValidPath ", path - await passWork(session) - let word = await passDaemonWord(session) - - of wopAddToStore: - assert session.version.minor >= 25 - let - name = await passClientString(session) - caMethod = await passClientString(session) - refs = await passClientStringSet(session) - repairBool = await passClientWord(session) - stderr.writeLine "wopAddToStore ", name - let n = await passClientChunks(session) - inc(chunksTotal, n) - await passWork(session) - let info = await passDaemonValidPathInfo(session, true) - - of wopAddTempRoot: - let path = await passClientString(session) - stderr.writeLine "wopAddTempRoot ", path - await passWork(session) - discard await passDaemonWord(session) - - of wopAddIndirectRoot: - let path = await passClientString(session) - stderr.writeLine "wopAddIndirectRoot ", path - await passWork(session) - discard await passDaemonWord(session) - - of wopSetOptions: - discard passClientWord(session) # keepFailed - discard passClientWord(session) # keepGoing - discard passClientWord(session) # tryFallback - discard passClientWord(session) # verbosity - discard passClientWord(session) # maxBuildJobs - discard passClientWord(session) # maxSilentTime - discard passClientWord(session) # useBuildHook - discard passClientWord(session) # verboseBuild - discard passClientWord(session) # logType - discard passClientWord(session) # printBuildTrace - discard passClientWord(session) # buildCores - discard passClientWord(session) # useSubstitutes - assert session.version.minor >= 12 - let overrides = await passClientStringMap(session) - await passWork(session) - - of wopQueryPathInfo: - assert session.version >= 17 - let path = await passClientString(session) - stderr.writeLine "wopQueryPathInfo ", path - await passWork(session) - let valid = await passDaemonWord(session) - if valid != 0: - var info = await passDaemonValidPathInfo(session, false) - info.path = path - stderr.writeLine "wopQueryPathInfo ", $info - - of wopQueryMissing: - assert session.version >= 30 - var miss: Missing - miss.targets = await passClientStringSeq(session) - await passWork(session) - miss.willBuild = await passDaemonStringSet(session) - miss.willSubstitute = await passDaemonStringSet(session) - miss.unknown = await passDaemonStringSet(session) - miss.downloadSize = BiggestInt await passDaemonWord(session) - miss.narSize = BiggestInt await passDaemonWord(session) - stderr.writeLine "wopQueryMissing ", $miss - - of wopBuildPathsWithResults: - assert session.version >= 34 - let - drvs = await passClientStringSeq(session) - buildMode = await passClientWord(session) - stderr.writeLine "wopBuildPathsWithResults drvs ", $drvs - await passWork(session) - let count = await passDaemonWord(session) - for _ in 1..count: - let - path = await passDaemonString(session) - status = await passDaemonWord(session) - errorMsg = await passDaemonString(session) - timesBUild = await passDaemonWord(session) - isNonDeterministic = await passDaemonWord(session) - startTime = await passDaemonWord(session) - stopTime = await passDaemonWord(session) - outputs = await passDaemonStringMap(session) - - else: - stderr.writeLine "unknown worker op ", wop.int - break - except ProtocolError as err: - stderr.writeLine "connection terminated" - stderr.writeLine "chunk bytes transfered: ", formatSize(chunksTotal) - finally: - close(session.daemon) - close(session.client) - -proc handshake(listener: AsyncSocket): Future[Snoop] {.async.} = - ## Take the next connection from `listener` and return a `Session`. - let session = Snoop(buffer: newSeq[Word](1024)) # 8KiB - session.client = await listener.accept() - session.daemon = newAsyncSocket( - domain = AF_UNIX, - sockType = SOCK_STREAM, - protocol = cast[Protocol](0), - buffered = false, - ) - await connectUnix(session.daemon, daemonSocketPath()) - let clientMagic = await passClientWord(session) - if clientMagic != WORKER_MAGIC_1: - raise newException(ProtocolError, "invalid protocol magic") - let daemonMagic = await passDaemonWord(session) - let daemonVersion = await passDaemonWord(session) - session.version = Version(await passClientWord(session)) - if session.version < PROTOCOL_VERSION: - raise newException(ProtocolError, "obsolete protocol version") - assert session.version.minor >= 14 - discard await(passClientWord(session)) - # obsolete CPU affinity - assert session.version.minor >= 11 - discard await(passClientWord(session)) - # obsolete reserveSpace - assert session.version.minor >= 33 - let daemonVersionString = await passDaemonString(session) - assert daemonVersionString == $store.nixVersion - await passWork(session) - return session - -proc emulateSocket*(path: string) {.async, gcsafe.} = - let listener = newAsyncSocket( - domain = AF_UNIX, - sockType = SOCK_STREAM, - protocol = cast[Protocol](0), - buffered = false) - bindUnix(listener, path) - listen(listener) - stderr.writeLine "listening on ", path - while not listener.isClosed: - try: - let session = await handshake(listener) - assert not session.isNil - asyncCheck loop(session) - except ProtocolError: - close(session) - finally: - close(session) diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 6f8755e7d986..09977b40a360 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -3,83 +3,39 @@ import preserves, std/tables, std/options type - Error* {.preservesRecord: "error".} = object - `message`*: Value - Eval* {.preservesRecord: "eval".} = object `expr`*: string `args`*: Value `result`* {.preservesEmbedded.}: EmbeddedRef - RepoArgsKind* {.pure.} = enum - `present`, `absent` - RepoArgsPresent* {.preservesDictionary.} = object - `args`*: Value + Error* {.preservesRecord: "error".} = object + `message`*: Value - RepoArgsAbsent* {.preservesDictionary.} = object + AttrSet* = Table[Symbol, Value] + LookupPathKind* {.pure.} = enum + `lookupPath`, `absent` + LookupPathLookupPath* {.preservesDictionary.} = object + `lookupPath`*: seq[string] + + LookupPathAbsent* {.preservesDictionary.} = object - `RepoArgs`* {.preservesOr.} = object - case orKind*: RepoArgsKind - of RepoArgsKind.`present`: - `present`*: RepoArgsPresent + `LookupPath`* {.preservesOr.} = object + case orKind*: LookupPathKind + of LookupPathKind.`lookupPath`: + `lookuppath`*: LookupPathLookupPath - of RepoArgsKind.`absent`: - `absent`*: RepoArgsAbsent + of LookupPathKind.`absent`: + `absent`*: LookupPathAbsent - RepoResolveStep* {.preservesRecord: "nix-repo".} = object - `detail`*: RepoResolveDetail - - AttrSet* = Table[Symbol, Value] Realise* {.preservesRecord: "realise".} = object `value`*: Value `result`* {.preservesEmbedded.}: EmbeddedRef - RepoStoreKind* {.pure.} = enum - `uri`, `cap`, `absent` - RepoStoreUri* {.preservesDictionary.} = object - `store`*: string - - RepoStoreCap* {.preservesDictionary.} = object - `store`* {.preservesEmbedded.}: EmbeddedRef - - RepoStoreAbsent* {.preservesDictionary.} = object - - `RepoStore`* {.preservesOr.} = object - case orKind*: RepoStoreKind - of RepoStoreKind.`uri`: - `uri`*: RepoStoreUri - - of RepoStoreKind.`cap`: - `cap`* {.preservesEmbedded.}: RepoStoreCap - - of RepoStoreKind.`absent`: - `absent`*: RepoStoreAbsent - - - RepoResolveDetailArgs* = Option[Value] - RepoResolveDetailCache* = Option[EmbeddedRef] - RepoResolveDetailImport* = string - RepoResolveDetailLookupPath* = seq[string] - RepoResolveDetailStore* = Option[Value] - `RepoResolveDetail`* {.preservesDictionary.} = object - `args`*: Option[Value] - `cache`*: Option[EmbeddedRef] - `import`*: string - `lookupPath`*: seq[string] - `store`*: Option[Value] - Derivation* {.preservesRecord: "drv".} = object `value`*: Value `context`*: Value - StoreResolveDetailCache* = Option[EmbeddedRef] - StoreResolveDetailUri* = string - `StoreResolveDetail`* {.preservesDictionary.} = object - `cache`*: Option[EmbeddedRef] - `params`*: AttrSet - `uri`*: string - ResultKind* {.pure.} = enum `Error`, `ok` ResultOk* {.preservesRecord: "ok".} = object @@ -94,11 +50,54 @@ type `ok`*: ResultOk - Context* = EmbeddedRef + StoreParamsKind* {.pure.} = enum + `storeParams`, `absent` + StoreParamsStoreParams* {.preservesDictionary.} = object + `storeParams`*: AttrSet + + StoreParamsAbsent* {.preservesDictionary.} = object + + `StoreParams`* {.preservesOr.} = object + case orKind*: StoreParamsKind + of StoreParamsKind.`storeParams`: + `storeparams`*: StoreParamsStoreParams + + of StoreParamsKind.`absent`: + `absent`*: StoreParamsAbsent + + + NixResolveStep* {.preservesRecord: "nix".} = object + `detail`*: NixResolveDetail + CheckStorePath* {.preservesRecord: "check-path".} = object `path`*: string `valid`* {.preservesEmbedded.}: EmbeddedRef + StoreUriKind* {.pure.} = enum + `storeUri`, `absent` + StoreUriStoreUri* {.preservesDictionary.} = object + `storeUri`*: string + + StoreUriAbsent* {.preservesDictionary.} = object + + `StoreUri`* {.preservesOr.} = object + case orKind*: StoreUriKind + of StoreUriKind.`storeUri`: + `storeuri`*: StoreUriStoreUri + + of StoreUriKind.`absent`: + `absent`*: StoreUriAbsent + + + NixResolveDetailCache* = Option[EmbeddedRef] + NixResolveDetailLookupPath* = Option[seq[string]] + NixResolveDetailStoreUri* = Option[string] + `NixResolveDetail`* {.preservesDictionary.} = object + `cache`*: Option[EmbeddedRef] + `lookupPath`*: Option[seq[string]] + `storeParams`*: Option[AttrSet] + `storeUri`*: Option[string] + CopyClosure* {.preservesRecord: "copy-closure".} = object `dest`* {.preservesEmbedded.}: EmbeddedRef `storePath`*: string @@ -120,31 +119,23 @@ type `absent`*: CacheSpaceAbsent - StoreResolveStep* {.preservesRecord: "nix-store".} = object - `detail`*: StoreResolveDetail - -proc `$`*(x: Error | Eval | RepoArgs | RepoResolveStep | AttrSet | Realise | - RepoStore | - RepoResolveDetail | - Derivation | - StoreResolveDetail | - Result | - Context | +proc `$`*(x: Eval | Error | AttrSet | LookupPath | Realise | Derivation | Result | + StoreParams | + NixResolveStep | CheckStorePath | + StoreUri | + NixResolveDetail | CopyClosure | - CacheSpace | - StoreResolveStep): string = + CacheSpace): string = `$`(toPreserves(x)) -proc encode*(x: Error | Eval | RepoArgs | RepoResolveStep | AttrSet | Realise | - RepoStore | - RepoResolveDetail | - Derivation | - StoreResolveDetail | +proc encode*(x: Eval | Error | AttrSet | LookupPath | Realise | Derivation | Result | - Context | + StoreParams | + NixResolveStep | CheckStorePath | + StoreUri | + NixResolveDetail | CopyClosure | - CacheSpace | - StoreResolveStep): seq[byte] = + CacheSpace): seq[byte] = encode(toPreserves(x)) diff --git a/test.pr b/test.pr index 8d6e6930d066..096724e11140 100755 --- a/test.pr +++ b/test.pr @@ -47,6 +47,7 @@ $log ! }> @"Service instantiation." + ? [ diff --git a/tests/test.nim b/tests/test.nim index b81049e9efdf..3349187f86c8 100644 --- a/tests/test.nim +++ b/tests/test.nim @@ -2,28 +2,34 @@ # SPDX-License-Identifier: Unlicense import - std/[unittest], + std/options, pkg/balls, + pkg/sys/ioqueue, pkg/preserves, + pkg/preserves/sugar, pkg/syndicate, - ../src/nix_actor/[nix_api, nix_values, protocol] + ../src/nix_actor/[nix_api, nix_values, nix_api_value, nix_api_expr] + +type Value = preserves.Value + +initLibstore() +initLibexpr() suite "libexpr": - initLibexpr() let store = openStore() - state = newState(store, []) + state = newState(store, ["nixpkgs"]) proc checkConversion(s: string) = runActor("checkConversion") do (turn: Turn): var nixVal = state.evalFromString(s, "") state.force(nixVal) nixVal.close() - var pr = state.toPreserves(nixVal) - checkpoint pr + var pr = nixVal.toPreserves(state) + checkpoint $pr var wirePr = turn.facet.exportNix(pr) - checkpoint wirePr + checkpoint $wirePr test "lists": let samples = [ @@ -48,7 +54,7 @@ suite "libexpr": test "derivation": let samples = [ - "let pkgs = import { }; in pkgs.hello" + "let depot = import /depot { }; in depot.users.emery.pkgs.syndicate-server" ] for s in samples: test s: @@ -65,34 +71,73 @@ type AddToStoreClientAttrs {.preservesDictionary.} = object eris: seq[byte] name: string -test "fromPreserve": +suite "fromPreserve": const raw = "{ca: > ca-method: |fixed:r:sha256| deriver: > eris: #[CgA1VVrR0k5gjgU1wKQKVZr1RkANf4zUva3vyc2wmLzhzuL8XqeUL0HE4W3aRpXNwXyFbaLxtXJiLCUWSyLjej+h] name: \"default-builder.sh\" narHash: > narSize: > references: [] registrationTime: > sigs: > ultimate: >}" let pr = parsePreserves(raw) var attrs: AddToStoreClientAttrs check fromPreserve(attrs, pr) -suite "gatekeeper": - - test "nix-repo": - var step: RepoResolveStep - - check not step.fromPreserves(parsePreserves""" - ", lookupPath: ["nixpkgs=/home/repo/nixpkgs/channel"] }> - """) - - check not step.fromPreserves(parsePreserves""" - - """) - - check step.fromPreserves parsePreserves""" - ", lookupPath: ["nixpkgs=/home/repo/nixpkgs/channel"] }> - """ - - check step.fromPreserves parsePreserves""" - ", lookupPath: ["nixpkgs=/home/repo/nixpkgs/channel"] }> - """ - - check step.fromPreserves parsePreserves""" - ", lookupPath: ["nixpkgs=/home/repo/nixpkgs/channel"], store: "local" }> - """ +suite "eval": + let + store = openStore() + eval = store.newState() + + test "function": + let + fn = eval.evalFromString("x: y: x + y") + x = (%"foo").toNix(eval) + y = (%"bar").toNix(eval) + checkpoint "fn:", fn.typeName + checkpoint " x:", x.typeName + checkpoint " y:", y.typeName + let + r = eval.apply(eval.apply(fn, x), y) + pr = r.toPreserves(eval).unthunkAll + + checkpoint $pr + check $pr == """"foobar"""" + + eval.close() + store.close() + +suite "import": + let + store = openStore() + eval = store.newState() + + block: + ## full expression string + let f = eval.evalFromString("false") + let t = eval.evalFromString("true") + let fs = $f.toPreserves(eval) + let ts = $t.toPreserves(eval) + checkpoint "false:", fs + check fs == "#f", fs + check ts == "#t", ts + + block: + ## full expression string + let fn = eval.evalFromString("x: x") + let x = eval.evalFromString("let pkgs = import /home/repo/nixpkgs {}; in pkgs.ncdc.meta.homepage") + eval.force(x) + let y = eval.apply(fn, x) + let pr = $y.toPreserves(eval).unthunkAll + checkpoint "$y.toPreserves(eval):", pr + check pr == """"https://dev.yorhel.nl/ncdc"""" + + block: + ## function + let pre = eval.evalFromString("import /home/repo/nixpkgs") + let args = eval.evalFromString("{ }") + let pkgs = eval.apply(pre, args) + let fn = eval.evalFromString("""pkgs: pkgs.ncdc.meta.homepage""") + let res = eval.apply(fn, pkgs) + let pr = res.toPreserves(eval).unthunkAll + assert not pr.isEmbedded + let text = $pr + checkpoint text + check text == """"https://dev.yorhel.nl/ncdc"""" + + eval.close() + store.close() -- cgit 1.4.1 From 0e35e937e47ce68c35ee6e997c81b2cd412bb32a Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 26 Aug 2024 05:06:54 +0300 Subject: Swap argument order at MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- protocol.prs | 7 +- sbom.json | 2 +- src/nix_actor.nim | 2 +- tests/test_eval_actor.nim | 191 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 197 insertions(+), 5 deletions(-) create mode 100644 tests/test_eval_actor.nim diff --git a/protocol.prs b/protocol.prs index c39a44a6853d..e9ad096989c0 100644 --- a/protocol.prs +++ b/protocol.prs @@ -39,9 +39,10 @@ Realise = . CopyClosure = . # Assertion. -# Eval at a nix-repo. @expr must be a function that takes two parameters, -# the first is the nix-repo value, the second is @args. -# The result is asserted to @result +# Eval at a nix capability. +# @expr must be a function that takes two parameters, +# the first is @args and the second is the value of +# the previous evaluation. The result is asserted to @result Eval = . AttrSet = {symbol: any ...:...} . diff --git a/sbom.json b/sbom.json index 4082821204a3..12deb59ddb58 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240825", + "version": "20240826", "authors": [ { "name": "Emery Hemingway" diff --git a/src/nix_actor.nim b/src/nix_actor.nim index e19b1397e251..2a2715296159 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -154,8 +154,8 @@ proc serve(entity: NixEntity; turn: Turn; r: Realise) = proc serve(entity: NixEntity; turn: Turn; e: Eval) = tryPublish(turn, e.result.Cap): var expr = entity.state.eval.evalFromString(e.expr) - expr = entity.state.eval.apply(expr, entity.root) expr = entity.state.eval.apply(expr, e.args.toNix(entity.state.eval)) + expr = entity.state.eval.apply(expr, entity.root) publishOk(turn, e.result.Cap, entity.newChild(turn, expr).self.toPreserves) method publish(entity: NixEntity; turn: Turn; a: AssertionRef; h: Handle) = diff --git a/tests/test_eval_actor.nim b/tests/test_eval_actor.nim new file mode 100644 index 000000000000..384ab58635c0 --- /dev/null +++ b/tests/test_eval_actor.nim @@ -0,0 +1,191 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +import + std/options, + pkg/balls, + pkg/sys/ioqueue, + pkg/preserves, + pkg/preserves/sugar, + pkg/syndicate, + pkg/syndicate/protocols/[gatekeeper, rpc], + ../src/nix_actor, + ../src/nix_actor/[nix_api, nix_values, protocol] + +type Value = preserves.Value + +initLibstore() +initLibexpr() + +type + ResultContinuation {.final.} = ref object of Entity + cb: proc (turn: Turn; v: Value) + +method publish(cont: ResultContinuation; turn: Turn; ass: AssertionRef; h: Handle) = + cont.cb(turn, ass.value) + +proc newResultContinuation[T](turn: Turn; cb: proc (turn: Turn; v: T)): Cap = + proc wrapper(turn: Turn; v: Value) = + var + err: ResultError + ok: ResultOk + if err.fromPreserves(v): + raiseAssert $err.error + check ok.fromPreserves(v) + var x = ok.value.preservesTo(T) + check x.isSome + if x.isSome: cb(turn, x.get) + turn.newCap(ResultContinuation(cb: wrapper)) + +suite "basic": + + var completed: bool + + proc actorTest(turn: Turn) = + turn.onStop do (turn: Turn): + block: + ## actor stopped + check completed + + checkpoint "actor booted" + let rootFacet = turn.facet + let ds = turn.newDataspace() + + let stepC = newResultContinuation(turn) do (turn: Turn; nix: Cap): + checkpoint "stepC" + block: + ## stepC + onPublish(turn, nix, grab()) do (v: Value): + checkpoint("stepC grabbed nix value " & $v) + assert not v.isRecord("null") + assert v == %"Hello VolgaSprint!" + completed = true + stop(rootFacet) + + let stepB = newResultContinuation(turn) do (turn: Turn; nix: Cap): + checkpoint "stepB" + block: + ## stepB + onPublish(turn, nix, grab()) do (v: Value): + checkpoint("stepB grabbed nix value " & $v) + assert not v.isRecord("null") + check v == %"Hello Volga" + publish(turn, nix, Eval( + expr: "y: x: x + y", + args: %"Sprint!", + result: stepC + )) + + let stepA = newResultContinuation(turn) do (turn: Turn; nix: Cap): + checkpoint "stepA" + block: + ## stepA + onPublish(turn, nix, grab()) do (v: Value): + checkpoint "stepA grabbed nix value " & $v + assert not v.isRecord("null") + check v == %"Hello" + publish(turn, nix, Eval( + expr: "y: x: x + y", + args: %" Volga", + result: stepB + )) + + during(turn, ds, ResolvedAccepted.grabWithin) do (nix: Cap): + checkpoint "resolve accepted" + block: + ## Resolved nix actor through gatekeeper + onPublish(turn, nix, grab()) do (v: Value): + checkpoint $v + publish(turn, nix, Eval( + expr: "y: x: y", + args: %"Hello", + result: stepA, + )) + + during(turn, ds, Rejected.grabType) do (rej: Rejected): + raiseAssert("resolve failed: " & $rej) + + publish(turn, ds, Resolve( + step: parsePreserves"""""", + observer: ds, + )) + + nix_actor.bootActor(turn, ds) + + block: + ## runActor + runActor("tests", actorTest) + check completed + +suite "nixpkgs": + + var completed: bool + + proc actorTest(turn: Turn) = + turn.onStop do (turn: Turn): + block: + ## actor stopped + check completed + + checkpoint "actor booted" + let rootFacet = turn.facet + let ds = turn.newDataspace() + + let stepC = newResultContinuation(turn) do (turn: Turn; nix: Cap): + checkpoint "stepC" + block: + ## stepC + onPublish(turn, nix, grab()) do (v: Value): + checkpoint("stepC grabbed nix value " & $v) + assert v == %"https://9fans.github.io/plan9port/" + completed = true + stop(rootFacet) + + let stepB = newResultContinuation(turn) do (turn: Turn; nix: Cap): + checkpoint "stepB" + block: + ## stepB + publish(turn, nix, Eval( + expr: "_: pkg: pkg.meta.homepage", + args: %false, + result: stepC + )) + + let stepA = newResultContinuation(turn) do (turn: Turn; nix: Cap): + checkpoint "stepA" + block: + ## stepA + publish(turn, nix, Eval( + expr: "builtins.getAttr", + args: %"plan9port", + result: stepB + )) + + during(turn, ds, ResolvedAccepted.grabWithin) do (nix: Cap): + checkpoint "resolve accepted" + block: + ## Resolved nix actor through gatekeeper + onPublish(turn, nix, grab()) do (v: Value): + checkpoint $v + publish(turn, nix, Eval( + expr: "args: _: import args", + args: initDictionary(), + result: stepA, + )) + + during(turn, ds, Rejected.grabType) do (rej: Rejected): + raiseAssert("resolve failed: " & $rej) + + publish(turn, ds, Resolve( + step: parsePreserves""" + + """, + observer: ds, + )) + + nix_actor.bootActor(turn, ds) + + block: + ## runActor + runActor("tests", actorTest) + check completed -- cgit 1.4.1 From 37882f0899a7812554ff3de983218b6541f80b80 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 26 Aug 2024 10:45:31 +0300 Subject: Simple after MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- protocol.prs | 14 +++++++++----- src/nix_actor.nim | 25 +++++++------------------ src/nix_actor/nix_values.nim | 11 +++++------ src/nix_actor/protocol.nim | 14 +++++++------- 4 files changed, 28 insertions(+), 36 deletions(-) diff --git a/protocol.prs b/protocol.prs index e9ad096989c0..bd1fe8a9feb9 100644 --- a/protocol.prs +++ b/protocol.prs @@ -32,19 +32,23 @@ CheckStorePath = . # Represents a Nix derivation. The @value can be realised via @context. Derivation = . -# Assertion. -Realise = . - # Assertion. The store that this asserted to will copy the closure of @storePath to @destination. When the copy completes or fails a Result value is asserted to @result. CopyClosure = . # Assertion. -# Eval at a nix capability. +# Eval at a Nix value capability. # @expr must be a function that takes two parameters, # the first is @args and the second is the value of -# the previous evaluation. The result is asserted to @result +# the previous evaluation. +# A new Nix value capability is return via @result. +# This capability can be observed using the Dataspace +# protocol but store paths may not be locally available. Eval = . +# Assertion. +# Realise a Nix value capability returned from to a string. +RealiseString = . + AttrSet = {symbol: any ...:...} . # Value. diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 2a2715296159..377bbc094bf4 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -7,7 +7,7 @@ import pkg/preserves/sugar, pkg/syndicate, pkg/syndicate/[gatekeepers, patterns, relays], - ./nix_actor/[nix_api, nix_api_value, nix_values, utils], + ./nix_actor/[nix_api, nix_values], ./nix_actor/protocol proc echo(args: varargs[string, `$`]) {.used.} = @@ -134,22 +134,10 @@ proc serve(entity: NixEntity; turn: Turn; obs: Observe) = captures[i] = initRecord("null") publish(turn, Cap obs.observer, captures) -proc serve(entity: NixEntity; turn: Turn; r: Realise) = +proc serve(entity: NixEntity; turn: Turn; r: RealiseString) = tryPublish(turn, r.result.Cap): - var drv: Derivation - if not drv.fromPreserves(r.value): - publishError(turn, r.result.Cap, %("failed to parse Derivation: " & $r.value)) - else: - var dummyCap = drv.context.unembed(Cap) - if dummyCap.isNone: - publishError(turn, r.result.Cap, %"derivation context is not a Cap") - else: - if not(dummyCap.get.target of NixValueRef): - publishError(turn, r.result.Cap, %"derivation context is not a NixValueRef") - else: - let v = entity.state.eval.realise(dummyCap.get.target.NixValueRef.value) - publishOk(turn, r.result.Cap, v) - # TODO: this is awkward. + publishOk(turn, r.result.Cap, + %entity.state.eval.realiseString(entity.root)) proc serve(entity: NixEntity; turn: Turn; e: Eval) = tryPublish(turn, e.result.Cap): @@ -160,12 +148,13 @@ proc serve(entity: NixEntity; turn: Turn; e: Eval) = method publish(entity: NixEntity; turn: Turn; a: AssertionRef; h: Handle) = var - # orc doesn't handle this as a union object + # TODO: this would be a union object + # but orc doesn't support it yet. checkPath: CheckStorePath copyClosure: CopyClosure eval: Eval observe: Observe - realise: Realise + realise: RealiseString if checkPath.fromPreserves(a.value): entity.serve(turn, checkPath) diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim index 8c87f51a9a97..a04c6d1578fb 100644 --- a/src/nix_actor/nix_values.nim +++ b/src/nix_actor/nix_values.nim @@ -213,15 +213,14 @@ proc step*(state: EvalState; nv: NixValue; path: openarray[preserves.Value]): Op result = nv.toPreserves(state, nix).some assert path.len > 0 or result.isSome -proc realise*(nix: NixContext; state: EvalState; val: NixValue): Value = - result = "".toPreserves +proc realiseString*(nix: NixContext; state: EvalState; val: NixValue): string = var rs = nix.string_realise(state, val, false) - result.string = newString(realised_string_get_buffer_size(rs)) - copyMem(result.string[0].addr, realised_string_get_buffer_start(rs), result.string.len) + result = newString(realised_string_get_buffer_size(rs)) + copyMem(result[0].addr, realised_string_get_buffer_start(rs), result.len) realised_string_free(rs) -proc realise*(state: EvalState; val: NixValue): Value = - mitNix: result = nix.realise(state, val) +proc realiseString*(state: EvalState; val: NixValue): string = + mitNix: result = nix.realiseString(state, val) proc initNull*(state: EvalState): NixValue = mitNix: diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 09977b40a360..325bc695e2d2 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -28,10 +28,6 @@ type `absent`*: LookupPathAbsent - Realise* {.preservesRecord: "realise".} = object - `value`*: Value - `result`* {.preservesEmbedded.}: EmbeddedRef - Derivation* {.preservesRecord: "drv".} = object `value`*: Value `context`*: Value @@ -69,6 +65,9 @@ type NixResolveStep* {.preservesRecord: "nix".} = object `detail`*: NixResolveDetail + RealiseString* {.preservesRecord: "realise-string".} = object + `result`* {.preservesEmbedded.}: EmbeddedRef + CheckStorePath* {.preservesRecord: "check-path".} = object `path`*: string `valid`* {.preservesEmbedded.}: EmbeddedRef @@ -119,9 +118,10 @@ type `absent`*: CacheSpaceAbsent -proc `$`*(x: Eval | Error | AttrSet | LookupPath | Realise | Derivation | Result | +proc `$`*(x: Eval | Error | AttrSet | LookupPath | Derivation | Result | StoreParams | NixResolveStep | + RealiseString | CheckStorePath | StoreUri | NixResolveDetail | @@ -129,10 +129,10 @@ proc `$`*(x: Eval | Error | AttrSet | LookupPath | Realise | Derivation | Result CacheSpace): string = `$`(toPreserves(x)) -proc encode*(x: Eval | Error | AttrSet | LookupPath | Realise | Derivation | - Result | +proc encode*(x: Eval | Error | AttrSet | LookupPath | Derivation | Result | StoreParams | NixResolveStep | + RealiseString | CheckStorePath | StoreUri | NixResolveDetail | -- cgit 1.4.1 From 64787e3811bfa875af7c273a82f64454b4175b0a Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 26 Aug 2024 12:39:09 +0300 Subject: Write errors to stderr to make escape codes readble --- src/nix_actor.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 377bbc094bf4..f191a52f2c61 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -18,6 +18,8 @@ type Value = preserves.Value template tryPublish(turn: Turn, cap: Cap; body: untyped) = try: body except CatchableError as err: + when not defined(release): + stderr.writeLine err.msg publish(turn, cap, Error(message: %err.msg)) proc publishOk(turn: Turn; cap: Cap, v: Value) = -- cgit 1.4.1 From f44684b2ed86e7c8bd0906dae6001b95789e92a6 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 27 Aug 2024 10:24:24 +0300 Subject: Do not convert Preserves sets to Nix --- sbom.json | 2 +- src/nix_actor.nim | 1 - src/nix_actor/nix_values.nim | 10 ++++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/sbom.json b/sbom.json index 12deb59ddb58..ee325b4b46d1 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240826", + "version": "20240827", "authors": [ { "name": "Emery Hemingway" diff --git a/src/nix_actor.nim b/src/nix_actor.nim index f191a52f2c61..8072b0c48c08 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -129,7 +129,6 @@ proc serve(entity: NixEntity; turn: Turn; obs: Observe) = break stepping for i, path in analysis.capturePaths: var v = entity.state.eval.step(entity.root, path) - assert v.isSome if v.isSome: captures[i] = turn.facet.exportNix(v.get) else: diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim index a04c6d1578fb..0b3cbee27fa9 100644 --- a/src/nix_actor/nix_values.nim +++ b/src/nix_actor/nix_values.nim @@ -156,16 +156,18 @@ proc translate*(nix: NixContext; state: EvalState; pr: preserves.Value): NixValu result = nixValRef.get.value else: raise newException(ValueError, "cannot convert Preserves record to Nix: " & $pr) - of pkSequence, pkSet: + of pkSequence: let b = nix.make_list_builder(state, pr.len.csize_t) defer: list_builder_free(b) - for i, e in pr: - checkError nix.list_builder_insert(b, i.register.cuint, nix.translate(state, e)) + for i, e in pr.sequence: + checkError nix.list_builder_insert(b, i.cuint, nix.translate(state, e)) checkError nix.make_list(b, result) + of pkSet: + raise newException(ValueError, "cannot convert Preserves sets to Nix") of pkDictionary: let b = nix.make_bindings_builder(state, pr.dict.len.csize_t) defer: bindings_builder_free(b) - for (name, value) in pr.dict: + for (name, value) in pr.pairs: if name.isSymbol: checkError nix.bindings_builder_insert(b, name.symbol.string, nix.translate(state, value)) else: -- cgit 1.4.1 From bc9c06f35112d2feb1f866e0633b3d005c36cf24 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 27 Aug 2024 10:24:37 +0300 Subject: Put eval capabilities in nested facets --- src/nix_actor.nim | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 8072b0c48c08..3ea60a96e8a4 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -82,10 +82,12 @@ proc newNixEntity(turn: Turn; detail: NixResolveDetail): NixEntity = proc newChild(parent: NixEntity; turn: Turn; val: NixValue): NixEntity = ## Create a child entity for a given root value. let entity = NixEntity(state: parent.state, root: val) - entity.state.eval.force(entity.root) - entity.self = newCap(turn, entity) - turn.onStop do (turn: Turn): - decref(entity.root) + turn.inFacet do (turn: Turn): + entity.state.eval.force(entity.root) + entity.facet = turn.facet + entity.self = newCap(turn, entity) + turn.onStop do (turn: Turn): + decref(entity.root) entity proc serve(entity: NixEntity; turn: Turn; checkPath: CheckStorePath) = -- cgit 1.4.1 From 012827ed7593e1f85499e8569d2537f860a22801 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 27 Aug 2024 10:28:41 +0300 Subject: Do not force Nix values until toPreserves --- src/nix_actor.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 3ea60a96e8a4..d213e472d910 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -83,7 +83,6 @@ proc newChild(parent: NixEntity; turn: Turn; val: NixValue): NixEntity = ## Create a child entity for a given root value. let entity = NixEntity(state: parent.state, root: val) turn.inFacet do (turn: Turn): - entity.state.eval.force(entity.root) entity.facet = turn.facet entity.self = newCap(turn, entity) turn.onStop do (turn: Turn): -- cgit 1.4.1 From d5b00581ba48d516da169e18f828d0061dd0edaf Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 27 Aug 2024 11:06:53 +0300 Subject: Flip order of again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/nix_actor.nim | 2 +- tests/test_eval_actor.nim | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/nix_actor.nim b/src/nix_actor.nim index d213e472d910..90f9b56b1b83 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -144,8 +144,8 @@ proc serve(entity: NixEntity; turn: Turn; r: RealiseString) = proc serve(entity: NixEntity; turn: Turn; e: Eval) = tryPublish(turn, e.result.Cap): var expr = entity.state.eval.evalFromString(e.expr) - expr = entity.state.eval.apply(expr, e.args.toNix(entity.state.eval)) expr = entity.state.eval.apply(expr, entity.root) + expr = entity.state.eval.apply(expr, e.args.toNix(entity.state.eval)) publishOk(turn, e.result.Cap, entity.newChild(turn, expr).self.toPreserves) method publish(entity: NixEntity; turn: Turn; a: AssertionRef; h: Handle) = diff --git a/tests/test_eval_actor.nim b/tests/test_eval_actor.nim index 384ab58635c0..f1e68577df43 100644 --- a/tests/test_eval_actor.nim +++ b/tests/test_eval_actor.nim @@ -71,7 +71,7 @@ suite "basic": assert not v.isRecord("null") check v == %"Hello Volga" publish(turn, nix, Eval( - expr: "y: x: x + y", + expr: "x: y: x + y", args: %"Sprint!", result: stepC )) @@ -85,7 +85,7 @@ suite "basic": assert not v.isRecord("null") check v == %"Hello" publish(turn, nix, Eval( - expr: "y: x: x + y", + expr: "x: y: x + y", args: %" Volga", result: stepB )) @@ -97,7 +97,7 @@ suite "basic": onPublish(turn, nix, grab()) do (v: Value): checkpoint $v publish(turn, nix, Eval( - expr: "y: x: y", + expr: "x: y: y", args: %"Hello", result: stepA, )) @@ -146,7 +146,7 @@ suite "nixpkgs": block: ## stepB publish(turn, nix, Eval( - expr: "_: pkg: pkg.meta.homepage", + expr: "pkg: _: pkg.meta.homepage", args: %false, result: stepC )) @@ -156,7 +156,7 @@ suite "nixpkgs": block: ## stepA publish(turn, nix, Eval( - expr: "builtins.getAttr", + expr: "pkgs: name: builtins.getAttr name pkgs", args: %"plan9port", result: stepB )) @@ -168,7 +168,7 @@ suite "nixpkgs": onPublish(turn, nix, grab()) do (v: Value): checkpoint $v publish(turn, nix, Eval( - expr: "args: _: import args", + expr: "_: args: import args", args: initDictionary(), result: stepA, )) -- cgit 1.4.1 From 76bb482c0bfe69b2bb86f06de472e801a1238013 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 27 Aug 2024 11:33:47 +0300 Subject: Update README, remove example config for now --- README.md | 23 +++++++++++++------ protocol.prs | 4 ---- test.pr | 65 ------------------------------------------------------ tests/nix-repo.pr | 29 ------------------------ tests/nix-store.pr | 37 ------------------------------- 5 files changed, 16 insertions(+), 142 deletions(-) delete mode 100755 test.pr delete mode 100644 tests/nix-repo.pr delete mode 100644 tests/nix-store.pr diff --git a/README.md b/README.md index 0ec4817be8b1..bc17cf9dd6c6 100644 --- a/README.md +++ b/README.md @@ -4,18 +4,27 @@ An actor for interacting with the [Nix](https://nixos.org/) daemon via the [Synd See [protocol.prs](./protocol.prs) for the Syndicate protocol [schema](https://preserves.dev/preserves-schema.html). -For an example configuration see [test.pr](./test.pr). +## Evaluation state as entity capabililties -## Expressions as dataspaces +The actor exposes on its initial capability a gatekeeper that resolves requests in the form ``. -The actor exposes on its initial capability a gatekeeper that resolves requests in the form ``. The resolved entity responds to observations as if it were a dataspace by asserting back lazily evaluated values from the imported expression. +The resolved entity is an evaluation state that responds to the assertions `` as well as observation of literal values via the dataspace protocol. +The evaluation state is initialized with the value `nil` and is advanced with Nix functions in the form of `prev: args: body` with a type of `Any -> Any -> Any`. -### Caveats -- Functions are not observable, unless the function can be satisfied with the empty attrset `{}` as an argument. -- An observation that stops at an attrset with an `outPath` yields the `outPath` and not the attrset. This prevents abritrary recursion into derivation dependencies. +To evaluate the `hello` package from Nixpkgs one could use an assertion like ` {})" "hello" #:…>` which would assert back a new assertion state at `hello`. + +The evaluation state represents a lazy Nix expression and must be "realised" to become a physical store path. +Asserting `` to an evaluation state will return a string with its realized closure at the evaluation store. + +With the exception of observations the result value of `` or `` is used for response assertions. + +Dataspace observations work over evaluation state. +In the example case of an evaluation state positioned at the `hello` package the observation of `{ hello: { meta: { license: { shortName: ? } } } }` would capture the value "gpl3Plus". +If an attrset contains an `outPath` attribute then the value of `outPath` is captured in place of the attrset. +This is to avoid traversing deeply nested and recursive Nix values. ### TODO -Realise store-paths from expressions using local and remote buils. +Realise store-paths from expressions using local and remote builds. ## Worker protocol diff --git a/protocol.prs b/protocol.prs index bd1fe8a9feb9..05f35544ef01 100644 --- a/protocol.prs +++ b/protocol.prs @@ -8,7 +8,6 @@ NixResolveDetail = { } & @lookupPath LookupPath & @storeUri StoreUri & @storeParams StoreParams -& @cacheSpace CacheSpace . # List of strings corresponding to entries in NIX_PATH. @@ -23,9 +22,6 @@ StoreUri = @storeUri { storeUri: string } / @absent { } . StoreParams = @storeParams { storeParams: AttrSet } / @absent { } . -# A cooperative caching dataspace shared with other Nix stores. -CacheSpace = @cacheSpace { cache: #:any } / @absent { } . - # Assertion. CheckStorePath = . diff --git a/test.pr b/test.pr deleted file mode 100755 index 096724e11140..000000000000 --- a/test.pr +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env -S syndicate-server --control --config - -let ?nix-actor = dataspace - -# $nix-actor -# ? ?any -# [ -# $log ! -# ?- [ $log ! ] -# ] - - $gatekeeper>> - $nix-actor #f> - -let ?nix-cache = dataspace - -let ?parent-route = ] - > - -let ?resolve-parent = - - -$log ! }> - -? > [ - $control ! -] - -? >> -[ - $parent += > - $parent - ? > - [ - @"Load test behavior." - $config += > - ] -] - -@"Service instantiation." - - - - -? -[ - $log ! - ? > - [ - let ?result = <* $config [ >> - >> - ]> ]> - - $log ! - $nix-actor $result> - ] -] diff --git a/tests/nix-repo.pr b/tests/nix-repo.pr deleted file mode 100644 index f012c12f0189..000000000000 --- a/tests/nix-repo.pr +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env -S syndicate-server --control --config - -? > -[ - let ?detail = { - store: $store - cache: $nix-cache - import: "" - lookupPath: [ - "nixpkgs=/home/repo/nixpkgs/channel" - ] - } - <-require-service > -] - -? > -[ - $repo - ? { plan9port: ?pkg } [ - $notifier - - $log ! - > ]>> - ] - $repo - ? { shapelib: { meta: { homepage: ?homepage } } } [ - $log ! - ] -] diff --git a/tests/nix-store.pr b/tests/nix-store.pr deleted file mode 100644 index 4b2e145c22a0..000000000000 --- a/tests/nix-store.pr +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env -S syndicate-server --config - -let ?trace = dataspace -$trace -? ?any -[ - $log ! - ?- [ $log ! ] -] - -let ?default-detail = { uri: "auto" params: {} cache: $nix-cache } -let ?test-detail = { label: "test" uri: "file:///tmp/nix-test-store" params: {} } - -# > - -? > -[ - $default-store ? ?any - [ $log ! ] - - let ?test-path = "/nix/store/90lh6zcbhqbil42yhfkx29rlnflxbz9v-cowsay-3.7.0" - $default-store += > ]> - > - - > - ? > - [ - $default-store += <-copy-closure - $test-store - $test-path - <* $notifier [ > ]> - > - ] - -] -- cgit 1.4.1 From 18d450ecc65232afccfeb1d1acb8ba6df886aa03 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 27 Aug 2024 12:04:05 +0300 Subject: Remove dead store code --- protocol.prs | 21 +++++++------------- src/nix_actor.nim | 45 ++++++------------------------------------ src/nix_actor/nix_values.nim | 33 ++----------------------------- src/nix_actor/protocol.nim | 47 ++++---------------------------------------- tests/test.nim | 2 +- 5 files changed, 20 insertions(+), 128 deletions(-) diff --git a/protocol.prs b/protocol.prs index 05f35544ef01..670df14826e6 100644 --- a/protocol.prs +++ b/protocol.prs @@ -2,6 +2,7 @@ version 1 . embeddedType EntityRef.Cap . # Gatekeeper step to access a Nix store. +# The resulting capability is a Nix evaluation state with an initial value of . NixResolveStep = . NixResolveDetail = { } @@ -23,26 +24,18 @@ StoreUri = @storeUri { storeUri: string } / @absent { } . StoreParams = @storeParams { storeParams: AttrSet } / @absent { } . # Assertion. -CheckStorePath = . - -# Represents a Nix derivation. The @value can be realised via @context. -Derivation = . - -# Assertion. The store that this asserted to will copy the closure of @storePath to @destination. When the copy completes or fails a Result value is asserted to @result. -CopyClosure = . - -# Assertion. -# Eval at a Nix value capability. +# Advance a Nix evaluation state. # @expr must be a function that takes two parameters, -# the first is @args and the second is the value of -# the previous evaluation. -# A new Nix value capability is return via @result. +# the first is the current evaluation value and the second +# is @args. A new Nix value capability is return via @result. # This capability can be observed using the Dataspace # protocol but store paths may not be locally available. Eval = . # Assertion. -# Realise a Nix value capability returned from to a string. +# Realise a Nix evaluation state to a string. +# This makes the closure of store paths referred to by the +# string present in the evaluation store. RealiseString = . AttrSet = {symbol: any ...:...} . diff --git a/src/nix_actor.nim b/src/nix_actor.nim index 90f9b56b1b83..b090899199be 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -71,6 +71,7 @@ type root: NixValue proc newNixEntity(turn: Turn; detail: NixResolveDetail): NixEntity = + ## Create an initial evaluation state. let entity = NixEntity(state: initState(detail)) entity.root = entity.state.eval.initNull() turn.onStop do (turn: Turn): @@ -89,35 +90,9 @@ proc newChild(parent: NixEntity; turn: Turn; val: NixValue): NixEntity = decref(entity.root) entity -proc serve(entity: NixEntity; turn: Turn; checkPath: CheckStorePath) = - tryPublish(turn, checkPath.valid.Cap): - let v = entity.state.store.isValidPath(checkPath.path) - publish(turn, checkPath.valid.Cap, initRecord("ok", %v)) - -proc serve(entity: NixEntity; turn: Turn; copy: CopyClosure) = - var dest = copy.dest.unembedEntity(NixEntity) - if dest.isNone: - publishError(turn, copy.result.Cap, %"destination store is not colocated with source store") - else: - tryPublish(turn, copy.result.Cap): - entity.state.store.copyClosure(dest.get.state.store, copy.storePath) - publishOk(turn, copy.result.Cap, %true) - # TODO: assert some stats or something. - proc serve(entity: NixEntity; turn: Turn; obs: Observe) = + ## Dataspace emulation. let facet = turn.facet - #[ - # TODO: move to a store entity - if obs.pattern.matches(initRecord("uri", %"")): - entity.state.store.getUri do (s: string): - facet.run do (turn: Turn): - publish(turn, obs.observer.Cap, obs.pattern.capture(initRecord("uri", %s)).get) - elif obs.pattern.matches(initRecord("version", %"")): - entity.state.store.getVersion do (s: string): - facet.run do (turn: Turn): - publish(turn, obs.observer.Cap, obs.pattern.capture(initRecord("version", %s)).get) - else: - ]# var analysis = analyse(obs.pattern) captures = newSeq[Value](analysis.capturePaths.len) @@ -131,15 +106,15 @@ proc serve(entity: NixEntity; turn: Turn; obs: Observe) = for i, path in analysis.capturePaths: var v = entity.state.eval.step(entity.root, path) if v.isSome: - captures[i] = turn.facet.exportNix(v.get) + captures[i] = v.get.unthunkAll else: captures[i] = initRecord("null") publish(turn, Cap obs.observer, captures) proc serve(entity: NixEntity; turn: Turn; r: RealiseString) = tryPublish(turn, r.result.Cap): - publishOk(turn, r.result.Cap, - %entity.state.eval.realiseString(entity.root)) + var str = entity.state.eval.realiseString(entity.root) + publishOk(turn, r.result.Cap, %str) proc serve(entity: NixEntity; turn: Turn; e: Eval) = tryPublish(turn, e.result.Cap): @@ -152,19 +127,11 @@ method publish(entity: NixEntity; turn: Turn; a: AssertionRef; h: Handle) = var # TODO: this would be a union object # but orc doesn't support it yet. - checkPath: CheckStorePath - copyClosure: CopyClosure eval: Eval observe: Observe realise: RealiseString - - if checkPath.fromPreserves(a.value): - entity.serve(turn, checkPath) - elif observe.fromPreserves(a.value): + if observe.fromPreserves(a.value): entity.serve(turn, observe) - elif copyClosure.fromPreserves(a.value) and - copyClosure.result of Cap: - entity.serve(turn, copyClosure) elif observe.fromPreserves(a.value) and observe.observer of Cap: serve(entity, turn, observe) elif realise.fromPreserves(a.value) and realise.result of Cap: diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim index 0b3cbee27fa9..854c44ef1c57 100644 --- a/src/nix_actor/nix_values.nim +++ b/src/nix_actor/nix_values.nim @@ -10,11 +10,9 @@ import type Value = preserves.Value NixValue* = nix_api.Value - NixValueRef* {.final.} = ref object of Entity - value*: NixValue StringThunkRef = ref StringThunkObj - StringThunkObj = object of EmbeddedObj + StringThunkObj {.final.} = object of EmbeddedObj data: Option[string] proc thunkString(start: cstring; n: cuint; state: pointer) {.cdecl.} = @@ -35,22 +33,6 @@ proc unthunk(v: Value): Value = proc unthunkAll*(v: Value): Value = v.mapEmbeds(unthunk) -proc exportNix*(facet: Facet; v: Value): Value = - proc op(v: Value): Value = - result = - if not v.isEmbeddedRef: v - else: - if v.embeddedRef of StringThunkRef: - var thunk = v.embeddedRef.StringThunkRef - if thunk.data.isSome: - thunk.data.get.toPreserves - else: v - elif v.embeddedRef of NixValueRef: - facet.newCap(v.embeddedRef.NixValueRef).embed - else: - v - v.mapEmbeds(op) - proc callThru(nix: NixContext; state: EvalState; nv: NixValue): NixValue = result = nv while true: @@ -97,10 +79,7 @@ proc toPreserves*(value: NixValue; state: EvalState; nix: NixContext): Value {.g result = str.toPreserves(state, nix) elif nix.has_attr_byname(value, state, "outPath"): var outPath = nix.get_attr_byname(value, state, "outPath") - result = Derivation( - value: outPath.toPreserves(state, nix), - context: NixValueRef(value: value).embed, - ).toPreserves + result = outPath.toPreserves(state, nix) else: let n = nix.getAttrsSize(value) result = initDictionary(int n) @@ -146,14 +125,6 @@ proc translate*(nix: NixContext; state: EvalState; pr: preserves.Value): NixValu of pkRecord: if pr.isRecord("null", 0): checkError nix.init_null(result) - elif pr.isRecord("drv", 2): - var drv: Derivation - if not drv.fromPreserves(pr): - raise newException(ValueError, "invalid derivation: " & $pr) - var nixValRef = drv.context.unembed(NixValueRef) - if not nixValRef.isSome: - raise newException(ValueError, "invalid Nix context: " & $drv.context) - result = nixValRef.get.value else: raise newException(ValueError, "cannot convert Preserves record to Nix: " & $pr) of pkSequence: diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 325bc695e2d2..92f1d65aacec 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -28,10 +28,6 @@ type `absent`*: LookupPathAbsent - Derivation* {.preservesRecord: "drv".} = object - `value`*: Value - `context`*: Value - ResultKind* {.pure.} = enum `Error`, `ok` ResultOk* {.preservesRecord: "ok".} = object @@ -68,10 +64,6 @@ type RealiseString* {.preservesRecord: "realise-string".} = object `result`* {.preservesEmbedded.}: EmbeddedRef - CheckStorePath* {.preservesRecord: "check-path".} = object - `path`*: string - `valid`* {.preservesEmbedded.}: EmbeddedRef - StoreUriKind* {.pure.} = enum `storeUri`, `absent` StoreUriStoreUri* {.preservesDictionary.} = object @@ -88,54 +80,23 @@ type `absent`*: StoreUriAbsent - NixResolveDetailCache* = Option[EmbeddedRef] NixResolveDetailLookupPath* = Option[seq[string]] NixResolveDetailStoreUri* = Option[string] `NixResolveDetail`* {.preservesDictionary.} = object - `cache`*: Option[EmbeddedRef] `lookupPath`*: Option[seq[string]] `storeParams`*: Option[AttrSet] `storeUri`*: Option[string] - CopyClosure* {.preservesRecord: "copy-closure".} = object - `dest`* {.preservesEmbedded.}: EmbeddedRef - `storePath`*: string - `result`* {.preservesEmbedded.}: EmbeddedRef - - CacheSpaceKind* {.pure.} = enum - `cacheSpace`, `absent` - CacheSpaceCacheSpace* {.preservesDictionary.} = object - `cache`* {.preservesEmbedded.}: EmbeddedRef - - CacheSpaceAbsent* {.preservesDictionary.} = object - - `CacheSpace`* {.preservesOr.} = object - case orKind*: CacheSpaceKind - of CacheSpaceKind.`cacheSpace`: - `cachespace`* {.preservesEmbedded.}: CacheSpaceCacheSpace - - of CacheSpaceKind.`absent`: - `absent`*: CacheSpaceAbsent - - -proc `$`*(x: Eval | Error | AttrSet | LookupPath | Derivation | Result | - StoreParams | +proc `$`*(x: Eval | Error | AttrSet | LookupPath | Result | StoreParams | NixResolveStep | RealiseString | - CheckStorePath | StoreUri | - NixResolveDetail | - CopyClosure | - CacheSpace): string = + NixResolveDetail): string = `$`(toPreserves(x)) -proc encode*(x: Eval | Error | AttrSet | LookupPath | Derivation | Result | - StoreParams | +proc encode*(x: Eval | Error | AttrSet | LookupPath | Result | StoreParams | NixResolveStep | RealiseString | - CheckStorePath | StoreUri | - NixResolveDetail | - CopyClosure | - CacheSpace): seq[byte] = + NixResolveDetail): seq[byte] = encode(toPreserves(x)) diff --git a/tests/test.nim b/tests/test.nim index 3349187f86c8..5e5fe23f5bac 100644 --- a/tests/test.nim +++ b/tests/test.nim @@ -28,7 +28,7 @@ suite "libexpr": nixVal.close() var pr = nixVal.toPreserves(state) checkpoint $pr - var wirePr = turn.facet.exportNix(pr) + var wirePr = pr.unthunkAll checkpoint $wirePr test "lists": -- cgit 1.4.1 From fa5a3462a78c268cc84d79cb748821fc123a3846 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 27 Aug 2024 12:17:38 +0300 Subject: Check for errors during realiseString --- src/nix_actor/nix_api_value.nim | 3 +++ src/nix_actor/nix_values.nim | 4 +++- src/nix_actor/utils.nim | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/nix_actor/nix_api_value.nim b/src/nix_actor/nix_api_value.nim index d4d5edc5f4c3..6c61f8883534 100644 --- a/src/nix_actor/nix_api_value.nim +++ b/src/nix_actor/nix_api_value.nim @@ -10,6 +10,9 @@ type ListBuilder* {.header: "nix_api_value.h", importc.} = distinct pointer RealisedString* {.header: "nix_api_value.h", importc: "nix_realised_string".} = distinct pointer +proc isNil*(p: BindingsBuilder|ExternalValue|ListBuilder|RealisedString): bool = + cast[pointer](p).isNil + proc alloc_value*(context: NixContext; state: EvalState): Value {.nix_api_value.} proc get_type*(context: NixContext; value: Value): ValueType {.nix_api_value.} diff --git a/src/nix_actor/nix_values.nim b/src/nix_actor/nix_values.nim index 854c44ef1c57..6eeadda66ae2 100644 --- a/src/nix_actor/nix_values.nim +++ b/src/nix_actor/nix_values.nim @@ -188,8 +188,10 @@ proc step*(state: EvalState; nv: NixValue; path: openarray[preserves.Value]): Op proc realiseString*(nix: NixContext; state: EvalState; val: NixValue): string = var rs = nix.string_realise(state, val, false) + if rs.isNil: raise newException(nix) result = newString(realised_string_get_buffer_size(rs)) - copyMem(result[0].addr, realised_string_get_buffer_start(rs), result.len) + if result.len > 0: + copyMem(result[0].addr, realised_string_get_buffer_start(rs), result.len) realised_string_free(rs) proc realiseString*(state: EvalState; val: NixValue): string = diff --git a/src/nix_actor/utils.nim b/src/nix_actor/utils.nim index 84b3e0ea65fb..ffe33b257b50 100644 --- a/src/nix_actor/utils.nim +++ b/src/nix_actor/utils.nim @@ -5,7 +5,7 @@ import ./nix_api_types, ./nix_api_util -proc newException(ctx: NixContext): ref NixException = +proc newException*(ctx: NixContext): ref NixException = new result var n: cuint -- cgit 1.4.1 From 35bca2edefe0808008e93d75447aa503a57153c0 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 27 Aug 2024 18:31:44 +0300 Subject: Restore the store entity and add replication --- README.md | 10 ++- protocol.prs | 20 ++++- sbom.json | 28 +++---- src/nix_actor.nim | 135 ++++++++++++++++++++++---------- src/nix_actor/protocol.nim | 81 ++++++++++++------- tests/test_actor.nim | 191 +++++++++++++++++++++++++++++++++++++++++++++ tests/test_eval_actor.nim | 191 --------------------------------------------- 7 files changed, 375 insertions(+), 281 deletions(-) create mode 100644 tests/test_actor.nim delete mode 100644 tests/test_eval_actor.nim diff --git a/README.md b/README.md index bc17cf9dd6c6..6a614c457adc 100644 --- a/README.md +++ b/README.md @@ -19,12 +19,16 @@ Asserting `` to an evaluation state will return With the exception of observations the result value of `` or `` is used for response assertions. Dataspace observations work over evaluation state. -In the example case of an evaluation state positioned at the `hello` package the observation of `{ hello: { meta: { license: { shortName: ? } } } }` would capture the value "gpl3Plus". +In the example case of an evaluation state positioned at the `hello` package the observation of `{ hello: { meta: { license: { shortName: ? } } } }` would capture the value `"gpl3Plus"`. If an attrset contains an `outPath` attribute then the value of `outPath` is captured in place of the attrset. This is to avoid traversing deeply nested and recursive Nix values. -### TODO -Realise store-paths from expressions using local and remote builds. +## Store replication + +Nix stores can be opened using the gatekeeper step ``. +The store entity responds to `` with true or false. + +To replicate paths between two stores, assert `` to a store entity or a evaluation entity, with `target` set to a store entity or a evaluation entity. ## Worker protocol diff --git a/protocol.prs b/protocol.prs index 670df14826e6..c3623234488b 100644 --- a/protocol.prs +++ b/protocol.prs @@ -2,10 +2,15 @@ version 1 . embeddedType EntityRef.Cap . # Gatekeeper step to access a Nix store. -# The resulting capability is a Nix evaluation state with an initial value of . -NixResolveStep = . +StoreResolveStep = . +StoreResolveDetail = { storeUri: string } +& @storeParams StoreParams +. -NixResolveDetail = { } +# Gatekeeper step to access a Nix evaluator. +# The resulting capability is a Nix evaluation state with an initial value of . +EvalResolveStep = . +EvalResolveDetail = { } & @lookupPath LookupPath & @storeUri StoreUri & @storeParams StoreParams @@ -38,6 +43,15 @@ Eval = . # string present in the evaluation store. RealiseString = . +# Assertion. +# Check at a Nix store if a store path is present and valid. +CheckStorePath = . + +# Assertion. +# Replicate a store path closure between stores. +# A Result value is asserted to @result. +Replicate = . + AttrSet = {symbol: any ...:...} . # Value. diff --git a/sbom.json b/sbom.json index ee325b4b46d1..eecbfd5ac73e 100644 --- a/sbom.json +++ b/sbom.json @@ -45,10 +45,10 @@ "type": "library", "bom-ref": "pkg:nim/syndicate", "name": "syndicate", - "version": "trunk", + "version": "20240827", "externalReferences": [ { - "url": "https://git.syndicate-lang.org/ehmry/syndicate-nim/archive/a2c723914b200e44e64ea67f4bf3a57b5d7e8a04.tar.gz", + "url": "https://git.syndicate-lang.org/ehmry/syndicate-nim/archive/1a445733561b64de448a9138cf30517df020f9d4.tar.gz", "type": "source-distribution" }, { @@ -63,23 +63,23 @@ }, { "name": "nix:fod:path", - "value": "/nix/store/vd1v0mdi7ww2rgy1s4s1hy3ll633lh4c-source" + "value": "/nix/store/mwxdy3sj4jfpaqmvj83rdhij405af1s9-source" }, { "name": "nix:fod:rev", - "value": "a2c723914b200e44e64ea67f4bf3a57b5d7e8a04" + "value": "1a445733561b64de448a9138cf30517df020f9d4" }, { "name": "nix:fod:sha256", - "value": "04f6rpz22xi308pg80lplsnpgg1l3lxwrj86krj4mng91kv8gs3c" + "value": "0ghm99jd5nzkrm35ml3w9l95ygd0l8wmpf7d2585j2rg53mfvhn7" }, { "name": "nix:fod:url", - "value": "https://git.syndicate-lang.org/ehmry/syndicate-nim/archive/a2c723914b200e44e64ea67f4bf3a57b5d7e8a04.tar.gz" + "value": "https://git.syndicate-lang.org/ehmry/syndicate-nim/archive/1a445733561b64de448a9138cf30517df020f9d4.tar.gz" }, { "name": "nix:fod:ref", - "value": "trunk" + "value": "20240827" }, { "name": "nix:fod:srcDir", @@ -129,10 +129,10 @@ "type": "library", "bom-ref": "pkg:nim/preserves", "name": "preserves", - "version": "trunk", + "version": "20240823", "externalReferences": [ { - "url": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/e22b354c11c6b9a87e3cde0e8958daa1414fb05b.tar.gz", + "url": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/4fe1952aafad9d852771549669077c0f3577267f.tar.gz", "type": "source-distribution" }, { @@ -147,23 +147,23 @@ }, { "name": "nix:fod:path", - "value": "/nix/store/lim0cw9vzy8pr31y3mvsxnc5xlbf6rnz-source" + "value": "/nix/store/b4cdj9x4y92a9i65j4mqcy89hpkhsjv0-source" }, { "name": "nix:fod:rev", - "value": "e22b354c11c6b9a87e3cde0e8958daa1414fb05b" + "value": "4fe1952aafad9d852771549669077c0f3577267f" }, { "name": "nix:fod:sha256", - "value": "1bskicgm7bx7sk0fig58h98xznzlnc0wqfbskhh0b5jsyzjrqi60" + "value": "00vs3m7kvz7lxmcz0zj3szqa82gdd3fc8i4qklhwr61cjiinb69w" }, { "name": "nix:fod:url", - "value": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/e22b354c11c6b9a87e3cde0e8958daa1414fb05b.tar.gz" + "value": "https://git.syndicate-lang.org/ehmry/preserves-nim/archive/4fe1952aafad9d852771549669077c0f3577267f.tar.gz" }, { "name": "nix:fod:ref", - "value": "trunk" + "value": "20240823" }, { "name": "nix:fod:srcDir", diff --git a/src/nix_actor.nim b/src/nix_actor.nim index b090899199be..8905330a8dd9 100644 --- a/src/nix_actor.nim +++ b/src/nix_actor.nim @@ -23,10 +23,15 @@ template tryPublish(turn: Turn, cap: Cap; body: untyped) = publish(turn, cap, Error(message: %err.msg)) proc publishOk(turn: Turn; cap: Cap, v: Value) = - publish(turn, cap, ResultOk(value: v)) + publish(turn, cap, protocol.ResultOk(value: v)) proc publishError(turn: Turn; cap: Cap, v: Value) = - publish(turn, cap, Error(message: v)) + publish(turn, cap, protocol.Error(message: v)) + +proc fromEmbedded[E](entity: var E; emb: EmbeddedRef): bool = + if emb of Cap and emb.Cap.target of E: + entity = emb.Cap.target.E + result = true proc unembedEntity(emb: EmbeddedRef; E: typedesc): Option[E] = if emb of Cap and emb.Cap.target of E: @@ -47,50 +52,89 @@ proc openStore(uri: string; params: Option[AttrSet]): Store = openStore(uri, pairs) type - NixState = object + NixEntity = ref object of Entity + self: Cap store: Store - eval: EvalState - -proc initState(detail: NixResolveDetail): NixState = - result.store = openStore(detail.storeUri.get("auto"), detail.storeParams) - if detail.lookupPath.isSome: - result.eval = newState(result.store, detail.lookupPath.get) - else: - result.eval = newState(result.store) -proc close(state: NixState) = - if not state.eval.isNil: - state.eval.close() - if not state.store.isNil: - state.store.close() + StoreEntity {.final.} = ref object of NixEntity -type - NixEntity {.acyclic, final.} = ref object of Entity - state: NixState - self: Cap + EvalEntity {.final.} = ref object of NixEntity + state: EvalState root: NixValue -proc newNixEntity(turn: Turn; detail: NixResolveDetail): NixEntity = +proc newStoreEntity(turn: Turn; detail: StoreResolveDetail): StoreEntity = + let entity = StoreEntity(store: openStore(detail.storeUri, detail.storeParams)) + entity.self = turn.newCap(entity) + entity.self.relay.onStop do (turn: Turn): + entity.store.close() + entity + +proc serve(entity: StoreEntity; turn: Turn; checkPath: CheckStorePath) = + tryPublish(turn, checkPath.valid.Cap): + var v = entity.store.isValidPath(checkPath.path) + publish(turn, checkPath.valid.Cap, initRecord("ok", %v)) + +proc serve(entity: NixEntity; turn: Turn; rep: Replicate) = + tryPublish(turn, rep.result.Cap): + var + target: Store + otherEntity = rep.target.unembedEntity(NixEntity) + if otherEntity.isSome: + target = otherEntity.get.store + if target.isNil: + publishError(turn, rep.result.Cap, %"cannot replicate with target") + else: + if entity.store.isValidPath(rep.storePath): + entity.store.copyClosure(target, rep.storePath) + # path exists at entity + else: + target.copyClosure(entity.store, rep.storePath) + # path hopefully exists at target + publishOk(turn, rep.result.Cap, %rep.storePath) + +method publish(entity: StoreEntity; turn: Turn; a: AssertionRef; h: Handle) = + var + # TODO: this would be a union object + # but orc doesn't support it yet. + checkStorePath: CheckStorePath + replicate: Replicate + if checkStorePath.fromPreserves(a.value): + entity.serve(turn, checkStorePath) + elif replicate.fromPreserves(a.value): + entity.serve(turn, replicate) + +proc newEvalEntity(turn: Turn; detail: EvalResolveDetail): EvalEntity = ## Create an initial evaluation state. - let entity = NixEntity(state: initState(detail)) - entity.root = entity.state.eval.initNull() - turn.onStop do (turn: Turn): + let entity = EvalEntity( + store: openStore(detail.storeUri.get("auto"), detail.storeParams) + ) + if detail.lookupPath.isSome: + entity.state = newState(entity.store, detail.lookupPath.get) + else: + entity.state = newState(entity.store) + entity.root = entity.state.initNull() + entity.self = turn.newCap(entity) + entity.self.relay.onStop do (turn: Turn): decref(entity.root) entity.state.close() - entity.self = newCap(turn, entity) + entity.store.close() entity -proc newChild(parent: NixEntity; turn: Turn; val: NixValue): NixEntity = +proc newChild(parent: EvalEntity; turn: Turn; val: NixValue): EvalEntity = ## Create a child entity for a given root value. - let entity = NixEntity(state: parent.state, root: val) + let entity = EvalEntity( + store: parent.store, + state: parent.state, + root: val + ) turn.inFacet do (turn: Turn): entity.facet = turn.facet entity.self = newCap(turn, entity) - turn.onStop do (turn: Turn): + entity.self.relay.onStop do (turn: Turn): decref(entity.root) entity -proc serve(entity: NixEntity; turn: Turn; obs: Observe) = +proc serve(entity: EvalEntity; turn: Turn; obs: Observe) = ## Dataspace emulation. let facet = turn.facet var @@ -98,46 +142,47 @@ proc serve(entity: NixEntity; turn: Turn; obs: Observe) = captures = newSeq[Value](analysis.capturePaths.len) block stepping: for i, path in analysis.constPaths: - var v = entity.state.eval.step(entity.root, path) + var v = entity.state.step(entity.root, path) if v.isNone or v.get != analysis.constValues[i]: let null = initRecord("null") for v in captures.mitems: v = null break stepping for i, path in analysis.capturePaths: - var v = entity.state.eval.step(entity.root, path) + var v = entity.state.step(entity.root, path) if v.isSome: captures[i] = v.get.unthunkAll else: captures[i] = initRecord("null") publish(turn, Cap obs.observer, captures) -proc serve(entity: NixEntity; turn: Turn; r: RealiseString) = +proc serve(entity: EvalEntity; turn: Turn; r: RealiseString) = tryPublish(turn, r.result.Cap): - var str = entity.state.eval.realiseString(entity.root) + var str = entity.state.realiseString(entity.root) publishOk(turn, r.result.Cap, %str) -proc serve(entity: NixEntity; turn: Turn; e: Eval) = +proc serve(entity: EvalEntity; turn: Turn; e: Eval) = tryPublish(turn, e.result.Cap): - var expr = entity.state.eval.evalFromString(e.expr) - expr = entity.state.eval.apply(expr, entity.root) - expr = entity.state.eval.apply(expr, e.args.toNix(entity.state.eval)) + var expr = entity.state.evalFromString(e.expr) + expr = entity.state.apply(expr, entity.root) + expr = entity.state.apply(expr, e.args.toNix(entity.state)) publishOk(turn, e.result.Cap, entity.newChild(turn, expr).self.toPreserves) -method publish(entity: NixEntity; turn: Turn; a: AssertionRef; h: Handle) = +method publish(entity: EvalEntity; turn: Turn; a: AssertionRef; h: Handle) = var # TODO: this would be a union object # but orc doesn't support it yet. eval: Eval observe: Observe realise: RealiseString - if observe.fromPreserves(a.value): - entity.serve(turn, observe) - elif observe.fromPreserves(a.value) and observe.observer of Cap: + replicate: Replicate + if observe.fromPreserves(a.value) and observe.observer of Cap: serve(entity, turn, observe) elif realise.fromPreserves(a.value) and realise.result of Cap: serve(entity, turn, realise) elif eval.fromPreserves(a.value) and eval.result of Cap: serve(entity, turn, eval) + elif replicate.fromPreserves(a.value) and replicate.result of Cap: + serve(entity, turn, replicate) else: when not defined(release): echo "unhandled assertion ", a.value @@ -147,8 +192,12 @@ proc bootActor*(turn: Turn; relay: Cap) = initLibexpr() let gk = spawnGatekeeper(turn, relay) - gk.serve do (turn: Turn; step: NixResolveStep) -> Resolved: - newNixEntity(turn, step.detail).self.resolveAccepted + + gk.serve do (turn: Turn; step: StoreResolveStep) -> rpc.Result: + newStoreEntity(turn, step.detail).self.resultOk + + gk.serve do (turn: Turn; step: EvalResolveStep) -> rpc.Result: + newEvalEntity(turn, step.detail).self.resultOk when isMainModule: runActor("main") do (turn: Turn): diff --git a/src/nix_actor/protocol.nim b/src/nix_actor/protocol.nim index 92f1d65aacec..bf5935e26670 100644 --- a/src/nix_actor/protocol.nim +++ b/src/nix_actor/protocol.nim @@ -3,6 +3,13 @@ import preserves, std/tables, std/options type + EvalResolveDetailLookupPath* = Option[seq[string]] + EvalResolveDetailStoreUri* = Option[string] + `EvalResolveDetail`* {.preservesDictionary.} = object + `lookupPath`*: Option[seq[string]] + `storeParams`*: Option[AttrSet] + `storeUri`*: Option[string] + Eval* {.preservesRecord: "eval".} = object `expr`*: string `args`*: Value @@ -28,20 +35,11 @@ type `absent`*: LookupPathAbsent - ResultKind* {.pure.} = enum - `Error`, `ok` - ResultOk* {.preservesRecord: "ok".} = object - `value`*: Value - - `Result`* {.preservesOr.} = object - case orKind*: ResultKind - of ResultKind.`Error`: - `error`*: Error - - of ResultKind.`ok`: - `ok`*: ResultOk + StoreResolveDetailStoreUri* = string + `StoreResolveDetail`* {.preservesDictionary.} = object + `storeParams`*: Option[AttrSet] + `storeUri`*: string - StoreParamsKind* {.pure.} = enum `storeParams`, `absent` StoreParamsStoreParams* {.preservesDictionary.} = object @@ -58,12 +56,27 @@ type `absent`*: StoreParamsAbsent - NixResolveStep* {.preservesRecord: "nix".} = object - `detail`*: NixResolveDetail + ResultKind* {.pure.} = enum + `Error`, `ok` + ResultOk* {.preservesRecord: "ok".} = object + `value`*: Value + + `Result`* {.preservesOr.} = object + case orKind*: ResultKind + of ResultKind.`Error`: + `error`*: Error + of ResultKind.`ok`: + `ok`*: ResultOk + + RealiseString* {.preservesRecord: "realise-string".} = object `result`* {.preservesEmbedded.}: EmbeddedRef + CheckStorePath* {.preservesRecord: "check-path".} = object + `path`*: string + `valid`* {.preservesEmbedded.}: EmbeddedRef + StoreUriKind* {.pure.} = enum `storeUri`, `absent` StoreUriStoreUri* {.preservesDictionary.} = object @@ -80,23 +93,37 @@ type `absent`*: StoreUriAbsent - NixResolveDetailLookupPath* = Option[seq[string]] - NixResolveDetailStoreUri* = Option[string] - `NixResolveDetail`* {.preservesDictionary.} = object - `lookupPath`*: Option[seq[string]] - `storeParams`*: Option[AttrSet] - `storeUri`*: Option[string] + Replicate* {.preservesRecord: "replicate".} = object + `target`* {.preservesEmbedded.}: EmbeddedRef + `storePath`*: string + `result`* {.preservesEmbedded.}: EmbeddedRef + + StoreResolveStep* {.preservesRecord: "nix-store".} = object + `detail`*: StoreResolveDetail + + EvalResolveStep* {.preservesRecord: "nix".} = object + `detail`*: EvalResolveDetail -proc `$`*(x: Eval | Error | AttrSet | LookupPath | Result | StoreParams | - NixResolveStep | +proc `$`*(x: EvalResolveDetail | Eval | Error | AttrSet | LookupPath | + StoreResolveDetail | + StoreParams | + Result | RealiseString | + CheckStorePath | StoreUri | - NixResolveDetail): string = + Replicate | + StoreResolveStep | + EvalResolveStep): string = `$`(toPreserves(x)) -proc encode*(x: Eval | Error | AttrSet | LookupPath | Result | StoreParams | - NixResolveStep | +proc encode*(x: EvalResolveDetail | Eval | Error | AttrSet | LookupPath | + StoreResolveDetail | + StoreParams | + Result | RealiseString | + CheckStorePath | StoreUri | - NixResolveDetail): seq[byte] = + Replicate | + StoreResolveStep | + EvalResolveStep): seq[byte] = encode(toPreserves(x)) diff --git a/tests/test_actor.nim b/tests/test_actor.nim new file mode 100644 index 000000000000..2a7aab92a2fa --- /dev/null +++ b/tests/test_actor.nim @@ -0,0 +1,191 @@ +# SPDX-FileCopyrightText: ☭ Emery Hemingway +# SPDX-License-Identifier: Unlicense + +import + std/options, + pkg/balls, + pkg/sys/ioqueue, + pkg/preserves, + pkg/preserves/sugar, + pkg/syndicate, + pkg/syndicate/protocols/[gatekeeper, rpc], + ../src/nix_actor, + ../src/nix_actor/[nix_api, nix_values, protocol] + +type Value = preserves.Value + +initLibstore() +initLibexpr() + +type + ResultContinuation {.final.} = ref object of Entity + cb: proc (turn: Turn; v: Value) + +method publish(cont: ResultContinuation; turn: Turn; ass: AssertionRef; h: Handle) = + cont.cb(turn, ass.value) + +proc newResultContinuation[T](turn: Turn; cb: proc (turn: Turn; v: T)): Cap = + proc wrapper(turn: Turn; v: Value) = + var + err: ResultError + ok: ResultOk + if err.fromPreserves(v): + raiseAssert $err.error + check ok.fromPreserves(v) + var x = ok.value.preservesTo(T) + check x.isSome + if x.isSome: cb(turn, x.get) + turn.newCap(ResultContinuation(cb: wrapper)) + +suite "basic": + + var completed: bool + + proc actorTest(turn: Turn) = + turn.onStop do (turn: Turn): + block: + ## actor stopped + check completed + + checkpoint "actor booted" + let rootFacet = turn.facet + let ds = turn.newDataspace() + + let stepC = newResultContinuation(turn) do (turn: Turn; nix: Cap): + checkpoint "stepC" + block: + ## stepC + onPublish(turn, nix, grab()) do (v: Value): + checkpoint("stepC grabbed nix value " & $v) + assert not v.isRecord("null") + assert v == %"Hello VolgaSprint!" + completed = true + stop(rootFacet) + + let stepB = newResultContinuation(turn) do (turn: Turn; nix: Cap): + checkpoint "stepB" + block: + ## stepB + onPublish(turn, nix, grab()) do (v: Value): + checkpoint("stepB grabbed nix value " & $v) + assert not v.isRecord("null") + check v == %"Hello Volga" + publish(turn, nix, Eval( + expr: "x: y: x + y", + args: %"Sprint!", + result: stepC + )) + + let stepA = newResultContinuation(turn) do (turn: Turn; nix: Cap): + checkpoint "stepA" + block: + ## stepA + onPublish(turn, nix, grab()) do (v: Value): + checkpoint "stepA grabbed nix value " & $v + assert not v.isRecord("null") + check v == %"Hello" + publish(turn, nix, Eval( + expr: "x: y: x + y", + args: %" Volga", + result: stepB + )) + + during(turn, ds, ResolvedAccepted.grabWithin) do (nix: Cap): + checkpoint "resolve accepted" + block: + ## Resolved nix actor through gatekeeper + onPublish(turn, nix, grab()) do (v: Value): + checkpoint $v + publish(turn, nix, Eval( + expr: "x: y: y", + args: %"Hello", + result: stepA, + )) + + during(turn, ds, Rejected.grabType) do (rej: Rejected): + raiseAssert("resolve failed: " & $rej) + + publish(turn, ds, Resolve( + step: parsePreserves"""""", + observer: ds, + )) + + nix_actor.bootActor(turn, ds) + + block: + ## runActor + runActor("tests", actorTest) + check completed + +suite "nixpkgs": + + var completed: bool + + proc actorTest(turn: Turn) = + turn.onStop do (turn: Turn): + block: + ## actor stopped + check completed + + checkpoint "actor booted" + let rootFacet = turn.facet + let ds = turn.newDataspace() + + let stepC = newResultContinuation(turn) do (turn: Turn; nix: Cap): + checkpoint "stepC" + block: + ## stepC + onPublish(turn, nix, grab()) do (v: Value): + checkpoint("stepC grabbed nix value " & $v) + assert v == %"https://9fans.github.io/plan9port/" + completed = true + stop(rootFacet) + + let stepB = newResultContinuation(turn) do (turn: Turn; nix: Cap): + checkpoint "stepB" + block: + ## stepB + publish(turn, nix, Eval( + expr: "pkg: _: pkg.meta.homepage", + args: %false, + result: stepC + )) + + let stepA = newResultContinuation(turn) do (turn: Turn; nix: Cap): + checkpoint "stepA" + block: + ## stepA + publish(turn, nix, Eval( + expr: "pkgs: name: builtins.getAttr name pkgs", + args: %"plan9port", + result: stepB + )) + + during(turn, ds, ResolvedAccepted.grabWithin) do (nix: Cap): + checkpoint "resolve accepted" + block: + ## Resolved nix actor through gatekeeper + onPublish(turn, nix, grab()) do (v: Value): + checkpoint $v + publish(turn, nix, Eval( + expr: "_: args: import args", + args: initDictionary(), + result: stepA, + )) + + during(turn, ds, Rejected.grabType) do (rej: Rejected): + raiseAssert("resolve failed: " & $rej) + + publish(turn, ds, Resolve( + step: parsePreserves""" + + """, + observer: ds, + )) + + nix_actor.bootActor(turn, ds) + + block: + ## runActor + runActor("tests", actorTest) + check completed diff --git a/tests/test_eval_actor.nim b/tests/test_eval_actor.nim deleted file mode 100644 index f1e68577df43..000000000000 --- a/tests/test_eval_actor.nim +++ /dev/null @@ -1,191 +0,0 @@ -# SPDX-FileCopyrightText: ☭ Emery Hemingway -# SPDX-License-Identifier: Unlicense - -import - std/options, - pkg/balls, - pkg/sys/ioqueue, - pkg/preserves, - pkg/preserves/sugar, - pkg/syndicate, - pkg/syndicate/protocols/[gatekeeper, rpc], - ../src/nix_actor, - ../src/nix_actor/[nix_api, nix_values, protocol] - -type Value = preserves.Value - -initLibstore() -initLibexpr() - -type - ResultContinuation {.final.} = ref object of Entity - cb: proc (turn: Turn; v: Value) - -method publish(cont: ResultContinuation; turn: Turn; ass: AssertionRef; h: Handle) = - cont.cb(turn, ass.value) - -proc newResultContinuation[T](turn: Turn; cb: proc (turn: Turn; v: T)): Cap = - proc wrapper(turn: Turn; v: Value) = - var - err: ResultError - ok: ResultOk - if err.fromPreserves(v): - raiseAssert $err.error - check ok.fromPreserves(v) - var x = ok.value.preservesTo(T) - check x.isSome - if x.isSome: cb(turn, x.get) - turn.newCap(ResultContinuation(cb: wrapper)) - -suite "basic": - - var completed: bool - - proc actorTest(turn: Turn) = - turn.onStop do (turn: Turn): - block: - ## actor stopped - check completed - - checkpoint "actor booted" - let rootFacet = turn.facet - let ds = turn.newDataspace() - - let stepC = newResultContinuation(turn) do (turn: Turn; nix: Cap): - checkpoint "stepC" - block: - ## stepC - onPublish(turn, nix, grab()) do (v: Value): - checkpoint("stepC grabbed nix value " & $v) - assert not v.isRecord("null") - assert v == %"Hello VolgaSprint!" - completed = true - stop(rootFacet) - - let stepB = newResultContinuation(turn) do (turn: Turn; nix: Cap): - checkpoint "stepB" - block: - ## stepB - onPublish(turn, nix, grab()) do (v: Value): - checkpoint("stepB grabbed nix value " & $v) - assert not v.isRecord("null") - check v == %"Hello Volga" - publish(turn, nix, Eval( - expr: "x: y: x + y", - args: %"Sprint!", - result: stepC - )) - - let stepA = newResultContinuation(turn) do (turn: Turn; nix: Cap): - checkpoint "stepA" - block: - ## stepA - onPublish(turn, nix, grab()) do (v: Value): - checkpoint "stepA grabbed nix value " & $v - assert not v.isRecord("null") - check v == %"Hello" - publish(turn, nix, Eval( - expr: "x: y: x + y", - args: %" Volga", - result: stepB - )) - - during(turn, ds, ResolvedAccepted.grabWithin) do (nix: Cap): - checkpoint "resolve accepted" - block: - ## Resolved nix actor through gatekeeper - onPublish(turn, nix, grab()) do (v: Value): - checkpoint $v - publish(turn, nix, Eval( - expr: "x: y: y", - args: %"Hello", - result: stepA, - )) - - during(turn, ds, Rejected.grabType) do (rej: Rejected): - raiseAssert("resolve failed: " & $rej) - - publish(turn, ds, Resolve( - step: parsePreserves"""""", - observer: ds, - )) - - nix_actor.bootActor(turn, ds) - - block: - ## runActor - runActor("tests", actorTest) - check completed - -suite "nixpkgs": - - var completed: bool - - proc actorTest(turn: Turn) = - turn.onStop do (turn: Turn): - block: - ## actor stopped - check completed - - checkpoint "actor booted" - let rootFacet = turn.facet - let ds = turn.newDataspace() - - let stepC = newResultContinuation(turn) do (turn: Turn; nix: Cap): - checkpoint "stepC" - block: - ## stepC - onPublish(turn, nix, grab()) do (v: Value): - checkpoint("stepC grabbed nix value " & $v) - assert v == %"https://9fans.github.io/plan9port/" - completed = true - stop(rootFacet) - - let stepB = newResultContinuation(turn) do (turn: Turn; nix: Cap): - checkpoint "stepB" - block: - ## stepB - publish(turn, nix, Eval( - expr: "pkg: _: pkg.meta.homepage", - args: %false, - result: stepC - )) - - let stepA = newResultContinuation(turn) do (turn: Turn; nix: Cap): - checkpoint "stepA" - block: - ## stepA - publish(turn, nix, Eval( - expr: "pkgs: name: builtins.getAttr name pkgs", - args: %"plan9port", - result: stepB - )) - - during(turn, ds, ResolvedAccepted.grabWithin) do (nix: Cap): - checkpoint "resolve accepted" - block: - ## Resolved nix actor through gatekeeper - onPublish(turn, nix, grab()) do (v: Value): - checkpoint $v - publish(turn, nix, Eval( - expr: "_: args: import args", - args: initDictionary(), - result: stepA, - )) - - during(turn, ds, Rejected.grabType) do (rej: Rejected): - raiseAssert("resolve failed: " & $rej) - - publish(turn, ds, Resolve( - step: parsePreserves""" - - """, - observer: ds, - )) - - nix_actor.bootActor(turn, ds) - - block: - ## runActor - runActor("tests", actorTest) - check completed -- cgit 1.4.1 From 4fb2792830dabc786b00524c9e0106017c663758 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 27 Aug 2024 18:44:11 +0300 Subject: Build with unreleased Nix library --- default.nix | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 9d278f027e40..93737413bd03 100644 --- a/default.nix +++ b/default.nix @@ -5,6 +5,15 @@ let inherit (pkgs) lib; buildNimSbom = pkgs.callPackage ./build-nim-sbom.nix { }; + nix' = pkgs.nixVersions.latest.overrideAttrs (_: { + version = "2024-08-23"; + src = pkgs.fetchFromGitHub { + owner = "nixos"; + repo = "nix"; + rev = "85f1aa6b3df5c5fcc924a74e2a9cc8acea9ba0e1"; + hash = "sha256-3+UgAktTtkGUNpxMxr+q+R+z3r026L3PwJzG6RD2IXM="; + }; + }); in buildNimSbom (finalAttrs: { outputs = [ @@ -12,7 +21,7 @@ buildNimSbom (finalAttrs: { "cfg" ]; nativeBuildInputs = [ pkgs.pkg-config ]; - buildInputs = [ pkgs.nixVersions.latest ]; + buildInputs = [ nix' ]; src = if lib.inNixShell then null else lib.cleanSource ./.; postInstall = '' mkdir $cfg -- cgit 1.4.1 From 280f52da984755ce6bd1b883d8afa4445c3334cc Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Wed, 28 Aug 2024 00:33:58 +0300 Subject: Make build-nim-sbom.nix compatible with nixos-24.05 --- build-nim-sbom.nix | 29 ++++++++++++++--------------- sbom.json | 2 +- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/build-nim-sbom.nix b/build-nim-sbom.nix index f1db25be7072..8edf4f98d60f 100644 --- a/build-nim-sbom.nix +++ b/build-nim-sbom.nix @@ -96,21 +96,14 @@ let install -Dt $out/bin ${output} '' ); -in - -callerArg: sbomArg: -let applySbom = + sbom: { passthru ? { }, ... }@prevAttrs: let - sbom = lib.attrsets.recursiveUpdate ( - if builtins.isAttrs sbomArg then sbomArg else builtins.fromJSON (builtins.readFile sbomArg) - ) passthru.sbom or { }; - properties = # SBOM metadata.component.properties as an attrset. lib.attrsets.recursiveUpdate (builtins.listToAttrs sbom.metadata.component.properties) passthru.properties or { }; @@ -177,20 +170,26 @@ let prevAttrs: { name, ... }@component: if (builtins.hasAttr name nimOverrides) then - prevAttrs // (nimOverrides.${name} component prevAttrs) + let + result = nimOverrides.${name} component prevAttrs; + in + prevAttrs // (if builtins.isAttrs result then result else result { }) else prevAttrs ) prevAttrs prevAttrs.passthru.sbom.components; - composition = - finalAttrs: + compose = + callerArg: sbom: finalAttrs: let callerAttrs = if builtins.isAttrs callerArg then callerArg else callerArg finalAttrs; - sbomAttrs = callerAttrs // (applySbom callerAttrs); + sbomAttrs = callerAttrs // (applySbom sbom callerAttrs); overrideAttrs = sbomAttrs // (applyOverrides sbomAttrs); in overrideAttrs; in -stdenv.mkDerivation composition - -# TODO: Add an overrideSbom function into the result.. +callerArg: sbomArg: +let + sbom = if builtins.isAttrs sbomArg then sbomArg else builtins.fromJSON (builtins.readFile sbomArg); + overrideSbom = f: stdenv.mkDerivation (compose callerArg (sbom // (f sbom))); +in +(stdenv.mkDerivation (compose callerArg sbom)) // { inherit overrideSbom; } diff --git a/sbom.json b/sbom.json index eecbfd5ac73e..567babc1e87d 100644 --- a/sbom.json +++ b/sbom.json @@ -7,7 +7,7 @@ "bom-ref": "pkg:nim/nix_actor", "name": "nix_actor", "description": "Syndicated Nix Actor", - "version": "20240827", + "version": "20240828", "authors": [ { "name": "Emery Hemingway" -- cgit 1.4.1 From d8606cc0e288a349bb4b9a436cdc7ef2feac34c5 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 2 Sep 2024 12:25:44 +0300 Subject: Make defaut.nix TVL depot compatible --- default.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 93737413bd03..901496264a3f 100644 --- a/default.nix +++ b/default.nix @@ -1,9 +1,10 @@ { pkgs ? import { }, + lib ? pkgs.lib, + ... }: let - inherit (pkgs) lib; buildNimSbom = pkgs.callPackage ./build-nim-sbom.nix { }; nix' = pkgs.nixVersions.latest.overrideAttrs (_: { version = "2024-08-23"; -- cgit 1.4.1