From 8c2084d0a1bbfc7d0bfa12ef1f5bda3cb02bff56 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sat, 27 Mar 2021 00:49:52 +0200 Subject: feat(tvix/proto): Add evaluator communication proto MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds the proto messages and service definition used for communicating with our evaluator. Please refer to the `Tvix - Component interaction` document for a detailed description of the interactions that this service is used for. Change-Id: If44cdbff66c6cfe88c22a748fe29c0bbd8510b0d Co-Authored-By: Florian Klink Co-Authored-By: Adam Höse Reviewed-on: https://cl.tvl.fyi/c/depot/+/2685 Tested-by: BuildkiteCI Reviewed-by: flokli Reviewed-by: grfn --- tvix/proto/LICENSE | 21 +++++++ tvix/proto/default.nix | 9 +++ tvix/proto/evaluator.proto | 144 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 tvix/proto/LICENSE create mode 100644 tvix/proto/default.nix create mode 100644 tvix/proto/evaluator.proto diff --git a/tvix/proto/LICENSE b/tvix/proto/LICENSE new file mode 100644 index 000000000000..36878fe4cbf1 --- /dev/null +++ b/tvix/proto/LICENSE @@ -0,0 +1,21 @@ +Copyright © 2021 The Tvix Authors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +“Software”), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/tvix/proto/default.nix b/tvix/proto/default.nix new file mode 100644 index 000000000000..7ff03ba3b74f --- /dev/null +++ b/tvix/proto/default.nix @@ -0,0 +1,9 @@ +# Build protocol buffer definitions to ensure that protos are valid in +# CI. Note that the output of this build target is not actually used +# anywhere, it just functions as a CI check for now. +{ pkgs, ... }: + +pkgs.runCommandNoCC "tvix-cc-proto" {} '' + mkdir $out + ${pkgs.protobuf}/bin/protoc -I ${./.} evaluator.proto --cpp_out=$out +'' diff --git a/tvix/proto/evaluator.proto b/tvix/proto/evaluator.proto new file mode 100644 index 000000000000..710a28fb9df7 --- /dev/null +++ b/tvix/proto/evaluator.proto @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: MIT +// Copyright © 2021 The Tvix Authors +syntax = "proto3"; + +package tvix.proto.evaluator.v1; + +service EvaluatorService { + rpc Evaluate(stream EvaluateRequest) returns (stream EvaluateResponse) {} +} + +// +// Message types for EvaluateRequest +// + +message EvaluateFile { + // Absolute path at which the evaluator can find the file to be + // evaluated. + string file_path = 1; + + // Optional attribute that should be evaluated within the file, + // assuming that the value it evaluates to is an attribute set. + optional string attribute = 2; + + // Additional arguments to pass into the evaluation, with which the + // file's top-level function will be auto-called. + map arguments = 3; +} + +message EvaluateExpression { + // Literal Nix expression to evaluate. + string expression = 1; + + // Working directory in which the expression should be evaluated. + string working_directory = 2; +} + +message BuildResultChunk { + string drv_hash = 1; + string output = 2; + bytes data = 3; + + // This field may be set on the first build result chunk returned + // to the evaluator, indicating the total size of the output that + // is going to be streamed in bytes. + // + // If set, the evaluator can use this to appropriately allocate a + // buffer for the output. + optional int64 output_size = 4; +} + +// Indicates that a single build has completed successfully. In case +// that the build outputs were required by the evaluator this also +// indicates that the output has been returned completely. +message BuildSuccess { + string drv_hash = 1; + string output = 2; +} + +// Describes an error that occured during a single build. +// +// TODO: We might want a more sophisticated error type. +message BuildError { + string drv_hash = 1; + string output = 2; + string error = 3; +} + +message BuildResult { + oneof build_result { + BuildSuccess build_success = 1; + BuildError build_error = 2; + } +} + + +/// Messages sent to the evaluator by the build coordinator. +message EvaluateRequest { + oneof message { + // Ask the evaluator to evaluate the specified file, and + // optionally attribute within that file. Must be the first + // message. + EvaluateFile evaluate_file = 1; + + // Ask the evaluator to evaluate the specified Nix expression. + // Must be the first message. + EvaluateExpression evaluate_expression = 2; + + // Send the chunks of a build result, in response to a + // BuildRequest. + // + // Note: This message might change as the store protocol is + // designed, as it is possible that mechanisms for transferring + // files might be reused between the protocols. + BuildResultChunk build_result_chunk = 3; + + // Indicate the result of a single build. See the documentation + // for the message types defined above for semantic details. + BuildResult build_result = 4; + } +} + +// +// Message types for EvaluateResponse +// + +// TODO: Placeholder type. +message Derivation { + string drv = 1; +} + +// TODO: Placeholder type. +message NixValue { + string value = 1; +} + +// TODO: Placeholder type. +message NixError { + string value = 1; +} + +message BuildRequest { + Derivation drv = 1; + string output = 2; +} + +// Messages returned to the coordinator by the evaluator. +message EvaluateResponse { + oneof message { + // A derivation that was instantiated while reducing the graph, + // and whose output is not required by the evaluator. + Derivation derivation = 1; + + // A derivation that was instantiated while reducing the graph, + // and whose output is required by the evaluator (IFD). + BuildRequest build_request = 2; + + // The final value yielded by the evaluation. Stream is closed + // after this. + NixValue done = 3; + + // Evaluation error. Stream is closed after this. + NixError error = 4; + } +} -- cgit 1.4.1