1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
|
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.
// Package builder implements the code required to build images via Nix. Image
// build data is cached for up to 24 hours to avoid duplicated calls to Nix
// (which are costly even if no building is performed).
package builder
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"os/exec"
"strings"
"cloud.google.com/go/storage"
"github.com/google/nixery/config"
)
// Image represents the information necessary for building a container image.
// This can be either a list of package names (corresponding to keys in the
// nixpkgs set) or a Nix expression that results in a *list* of derivations.
type Image struct {
Name string
Tag string
// Names of packages to include in the image. These must correspond
// directly to top-level names of Nix packages in the nixpkgs tree.
Packages []string
}
// ImageFromName parses an image name into the corresponding structure which can
// be used to invoke Nix.
//
// It will expand convenience names under the hood (see the `convenienceNames`
// function below).
func ImageFromName(name string, tag string) Image {
packages := strings.Split(name, "/")
return Image{
Name: name,
Tag: tag,
Packages: convenienceNames(packages),
}
}
// BuildResult represents the output of calling the Nix derivation responsible
// for building registry images.
//
// The `layerLocations` field contains the local filesystem paths to each
// individual image layer that will need to be served, while the `manifest`
// field contains the JSON-representation of the manifest that needs to be
// served to the client.
//
// The later field is simply treated as opaque JSON and passed through.
type BuildResult struct {
Error string `json:"error"`
Pkgs []string `json:"pkgs"`
Manifest json.RawMessage `json:"manifest"`
LayerLocations map[string]struct {
Path string `json:"path"`
Md5 []byte `json:"md5"`
} `json:"layerLocations"`
}
// convenienceNames expands convenience package names defined by Nixery which
// let users include commonly required sets of tools in a container quickly.
//
// Convenience names must be specified as the first package in an image.
//
// Currently defined convenience names are:
//
// * `shell`: Includes bash, coreutils and other common command-line tools
func convenienceNames(packages []string) []string {
shellPackages := []string{"bashInteractive", "coreutils", "moreutils", "nano"}
if packages[0] == "shell" {
return append(packages[1:], shellPackages...)
}
return packages
}
// Call out to Nix and request that an image be built. Nix will, upon success,
// return a manifest for the container image.
func BuildImage(ctx *context.Context, cfg *config.Config, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) {
packages, err := json.Marshal(image.Packages)
if err != nil {
return nil, err
}
args := []string{
"--argstr", "name", image.Name,
"--argstr", "packages", string(packages),
}
if cfg.Pkgs != nil {
args = append(args, "--argstr", "pkgSource", cfg.Pkgs.Render(image.Tag))
}
cmd := exec.Command("nixery-build-image", args...)
outpipe, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
errpipe, err := cmd.StderrPipe()
if err != nil {
return nil, err
}
if err = cmd.Start(); err != nil {
log.Println("Error starting nix-build:", err)
return nil, err
}
log.Printf("Started Nix image build for '%s'", image.Name)
stdout, _ := ioutil.ReadAll(outpipe)
stderr, _ := ioutil.ReadAll(errpipe)
if err = cmd.Wait(); err != nil {
// TODO(tazjin): Propagate errors upwards in a usable format.
log.Printf("nix-build execution error: %s\nstdout: %s\nstderr: %s\n", err, stdout, stderr)
return nil, err
}
log.Println("Finished Nix image build")
buildOutput, err := ioutil.ReadFile(strings.TrimSpace(string(stdout)))
if err != nil {
return nil, err
}
// The build output returned by Nix is deserialised to add all
// contained layers to the bucket. Only the manifest itself is
// re-serialised to JSON and returned.
var result BuildResult
err = json.Unmarshal(buildOutput, &result)
if err != nil {
return nil, err
}
for layer, meta := range result.LayerLocations {
err = uploadLayer(ctx, bucket, layer, meta.Path, meta.Md5)
if err != nil {
return nil, err
}
}
return &result, nil
}
// uploadLayer uploads a single layer to Cloud Storage bucket. Before writing
// any data the bucket is probed to see if the file already exists.
//
// If the file does exist, its MD5 hash is verified to ensure that the stored
// file is not - for example - a fragment of a previous, incomplete upload.
func uploadLayer(ctx *context.Context, bucket *storage.BucketHandle, layer string, path string, md5 []byte) error {
layerKey := fmt.Sprintf("layers/%s", layer)
obj := bucket.Object(layerKey)
// Before uploading a layer to the bucket, probe whether it already
// exists.
//
// If it does and the MD5 checksum matches the expected one, the layer
// upload can be skipped.
attrs, err := obj.Attrs(*ctx)
if err == nil && bytes.Equal(attrs.MD5, md5) {
log.Printf("Layer sha256:%s already exists in bucket, skipping upload", layer)
} else {
writer := obj.NewWriter(*ctx)
file, err := os.Open(path)
if err != nil {
return fmt.Errorf("failed to open layer %s from path %s: %v", layer, path, err)
}
size, err := io.Copy(writer, file)
if err != nil {
return fmt.Errorf("failed to write layer %s to Cloud Storage: %v", layer, err)
}
if err = writer.Close(); err != nil {
return fmt.Errorf("failed to write layer %s to Cloud Storage: %v", layer, err)
}
log.Printf("Uploaded layer sha256:%s (%v bytes written)\n", layer, size)
}
return nil
}
|
