about summary refs log tree commit diff
path: root/tools/nixery/main.go
blob: 044d791517bcd3173072ac5f25b17d29a07af7b1 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
// Package main provides the implementation of a container registry that
// transparently builds container images based on Nix derivations.
//
// The Nix derivation used for image creation is responsible for creating
// objects that are compatible with the registry API. The targeted registry
// protocol is currently Docker's.
//
// When an image is requested, the required contents are parsed out of the
// request and a Nix-build is initiated that eventually responds with the
// manifest as well as information linking each layer digest to a local
// filesystem path.
//
// Nixery caches the filesystem paths and returns the manifest to the client.
// Subsequent requests for layer content per digest are then fulfilled by
// serving the files from disk.
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"cloud.google.com/go/storage"
)

// config holds the Nixery configuration options.
type config struct {
	bucket  string // GCS bucket to cache & serve layers
	builder string // Nix derivation for building images
	web     string // Static files to serve over HTTP
	port    string // Port on which to launch HTTP server
}

// ManifestMediaType is the Content-Type used for the manifest itself. This
// corresponds to the "Image Manifest V2, Schema 2" described on this page:
//
// https://docs.docker.com/registry/spec/manifest-v2-2/
const manifestMediaType string = "application/vnd.docker.distribution.manifest.v2+json"

// Image represents the information necessary for building a container image.
// This can be either a list of package names (corresponding to keys in the
// nixpkgs set) or a Nix expression that results in a *list* of derivations.
type image struct {
	// Name of the container image.
	name string

	// Names of packages to include in the image. These must correspond
	// directly to top-level names of Nix packages in the nixpkgs tree.
	packages []string
}

// BuildResult represents the output of calling the Nix derivation responsible
// for building registry images.
//
// The `layerLocations` field contains the local filesystem paths to each
// individual image layer that will need to be served, while the `manifest`
// field contains the JSON-representation of the manifest that needs to be
// served to the client.
//
// The later field is simply treated as opaque JSON and passed through.
type BuildResult struct {
	Manifest       json.RawMessage `json:"manifest"`
	LayerLocations map[string]struct {
		Path string `json:"path"`
		Md5  []byte `json:"md5"`
	} `json:"layerLocations"`
}

// imageFromName parses an image name into the corresponding structure which can
// be used to invoke Nix.
//
// It will expand convenience names under the hood (see the `convenienceNames`
// function below).
func imageFromName(name string) image {
	packages := strings.Split(name, "/")
	return image{
		name:     name,
		packages: convenienceNames(packages),
	}
}

// convenienceNames expands convenience package names defined by Nixery which
// let users include commonly required sets of tools in a container quickly.
//
// Convenience names must be specified as the first package in an image.
//
// Currently defined convenience names are:
//
// * `shell`: Includes bash, coreutils and other common command-line tools
// * `builder`: All of the above and the standard build environment
func convenienceNames(packages []string) []string {
	shellPackages := []string{"bashInteractive", "coreutils", "moreutils", "nano"}
	builderPackages := append(shellPackages, "stdenv")

	if packages[0] == "shell" {
		return append(packages[1:], shellPackages...)
	} else if packages[0] == "builder" {
		return append(packages[1:], builderPackages...)
	} else {
		return packages
	}
}

// Call out to Nix and request that an image be built. Nix will, upon success,
// return a manifest for the container image.
func buildImage(ctx *context.Context, cfg *config, image *image, bucket *storage.BucketHandle) ([]byte, error) {
	packages, err := json.Marshal(image.packages)
	if err != nil {
		return nil, err
	}

	cmd := exec.Command(
		"nix-build",
		"--no-out-link",
		"--show-trace",
		"--argstr", "name", image.name,
		"--argstr", "packages", string(packages), cfg.builder,
	)

	outpipe, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	errpipe, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}

	if err = cmd.Start(); err != nil {
		log.Println("Error starting nix-build:", err)
		return nil, err
	}
	log.Printf("Started Nix image build for '%s'", image.name)

	stdout, _ := ioutil.ReadAll(outpipe)
	stderr, _ := ioutil.ReadAll(errpipe)

	if err = cmd.Wait(); err != nil {
		// TODO(tazjin): Propagate errors upwards in a usable format.
		log.Printf("nix-build execution error: %s\nstdout: %s\nstderr: %s\n", err, stdout, stderr)
		return nil, err
	}

	log.Println("Finished Nix image build")

	buildOutput, err := ioutil.ReadFile(strings.TrimSpace(string(stdout)))
	if err != nil {
		return nil, err
	}

	// The build output returned by Nix is deserialised to add all
	// contained layers to the bucket. Only the manifest itself is
	// re-serialised to JSON and returned.
	var result BuildResult
	err = json.Unmarshal(buildOutput, &result)
	if err != nil {
		return nil, err
	}

	for layer, meta := range result.LayerLocations {
		err = uploadLayer(ctx, bucket, layer, meta.Path, meta.Md5)
		if err != nil {
			return nil, err
		}
	}

	return json.Marshal(result.Manifest)
}

// uploadLayer uploads a single layer to Cloud Storage bucket. Before writing
// any data the bucket is probed to see if the file already exists.
//
// If the file does exist, its MD5 hash is verified to ensure that the stored
// file is not - for example - a fragment of a previous, incomplete upload.
func uploadLayer(ctx *context.Context, bucket *storage.BucketHandle, layer string, path string, md5 []byte) error {
	layerKey := fmt.Sprintf("layers/%s", layer)
	obj := bucket.Object(layerKey)

	// Before uploading a layer to the bucket, probe whether it already
	// exists.
	//
	// If it does and the MD5 checksum matches the expected one, the layer
	// upload can be skipped.
	attrs, err := obj.Attrs(*ctx)

	if err == nil && bytes.Equal(attrs.MD5, md5) {
		log.Printf("Layer sha256:%s already exists in bucket, skipping upload", layer)
	} else {
		writer := obj.NewWriter(*ctx)
		file, err := os.Open(path)

		if err != nil {
			return fmt.Errorf("failed to open layer %s from path %s: %v", layer, path, err)
		}

		size, err := io.Copy(writer, file)
		if err != nil {
			return fmt.Errorf("failed to write layer %s to Cloud Storage: %v", layer, err)
		}

		if err = writer.Close(); err != nil {
			return fmt.Errorf("failed to write layer %s to Cloud Storage: %v", layer, err)
		}

		log.Printf("Uploaded layer sha256:%s (%v bytes written)\n", layer, size)
	}

	return nil
}

// layerRedirect constructs the public URL of the layer object in the Cloud
// Storage bucket and redirects the client there.
//
// The Docker client is known to follow redirects, but this might not be true
// for all other registry clients.
func layerRedirect(w http.ResponseWriter, cfg *config, digest string) {
	log.Printf("Redirecting layer '%s' request to bucket '%s'\n", digest, cfg.bucket)
	url := fmt.Sprintf("https://storage.googleapis.com/%s/layers/%s", cfg.bucket, digest)
	w.Header().Set("Location", url)
	w.WriteHeader(303)
}

// prepareBucket configures the handle to a Cloud Storage bucket in which
// individual layers will be stored after Nix builds. Nixery does not directly
// serve layers to registry clients, instead it redirects them to the public
// URLs of the Cloud Storage bucket.
//
// The bucket is required for Nixery to function correctly, hence fatal errors
// are generated in case it fails to be set up correctly.
func prepareBucket(ctx *context.Context, cfg *config) *storage.BucketHandle {
	client, err := storage.NewClient(*ctx)
	if err != nil {
		log.Fatalln("Failed to set up Cloud Storage client:", err)
	}

	bkt := client.Bucket(cfg.bucket)

	if _, err := bkt.Attrs(*ctx); err != nil {
		log.Fatalln("Could not access configured bucket", err)
	}

	return bkt
}

var manifestRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/manifests/(\w+)$`)
var layerRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/blobs/sha256:(\w+)$`)

func getConfig(key, desc string) string {
	value := os.Getenv(key)
	if value == "" {
		log.Fatalln(desc + " must be specified")
	}

	return value
}

func main() {
	cfg := &config{
		bucket: getConfig("BUCKET", "GCS bucket for layer storage"),
		builder: getConfig("NIX_BUILDER", "Nix image builder code"),
		web: getConfig("WEB_DIR", "Static web file dir"),
		port: getConfig("PORT", "HTTP port"),
	}

	ctx := context.Background()
	bucket := prepareBucket(&ctx, cfg)

	log.Printf("Starting Kubernetes Nix controller on port %s\n", cfg.port)

	log.Fatal(http.ListenAndServe(":"+cfg.port, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// When running on AppEngine, HTTP traffic should be redirected
		// to HTTPS.
		//
		// This is achieved here by enforcing HSTS (with a one week
		// duration) on responses.
		if r.Header.Get("X-Forwarded-Proto") == "http" && strings.Contains(r.Host, "appspot.com") {
			w.Header().Add("Strict-Transport-Security", "max-age=604800")
		}

		// Serve an index page to anyone who visits the registry's base
		// URL:
		if r.RequestURI == "/" {
			index, _ := ioutil.ReadFile(cfg.web + "/index.html")
			w.Header().Add("Content-Type", "text/html")
			w.Write(index)
			return
		}

		// Acknowledge that we speak V2
		if r.RequestURI == "/v2/" {
			fmt.Fprintln(w)
			return
		}

		// Serve the manifest (straight from Nix)
		manifestMatches := manifestRegex.FindStringSubmatch(r.RequestURI)
		if len(manifestMatches) == 3 {
			imageName := manifestMatches[1]
			log.Printf("Requesting manifest for image '%s'", imageName)
			image := imageFromName(manifestMatches[1])
			manifest, err := buildImage(&ctx, cfg, &image, bucket)

			if err != nil {
				log.Println("Failed to build image manifest", err)
				return
			}

			w.Header().Add("Content-Type", manifestMediaType)
			w.Write(manifest)
			return
		}

		// Serve an image layer. For this we need to first ask Nix for
		// the manifest, then proceed to extract the correct layer from
		// it.
		layerMatches := layerRegex.FindStringSubmatch(r.RequestURI)
		if len(layerMatches) == 3 {
			digest := layerMatches[2]
			layerRedirect(w, cfg, digest)
			return
		}

		w.WriteHeader(404)
	})))
}