blob: e87189da2a89ce453e7d701bd7a2c9855e873b2b (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
{ config, depot, pkgs, ... }:
{
imports = [ ./nar-bridge-module.nix ];
# Microbenchmark
# hyperfine --warmup 1 'rm -rf /tmp/cache; nix copy --from https://nixos.tvix.store/ --to "file:///tmp/cache?compression=none" /nix/store/jlkypcf54nrh4n6r0l62ryx93z752hb2-firefox-132.0'
# From a different hetzner machine with 1Gbps uplink:
# - with zstd: 13.384s
# - with gzip: 11.130s
# - with brotli: ~18s
# - without compression: 15.6s
# From a 1Gbit link in TUM:
# - with zstd: 32.292s
# - with gzip: 51s
# - cache.nixos.org from the same connection: 36.559s
services.nginx = {
package = pkgs.nginxStable.override {
modules = [ pkgs.nginxModules.zstd ];
};
virtualHosts.${config.machine.domain} = {
# when using http2 we actually see worse throughput,
# because it only uses a single tcp connection,
# which pins nginx to a single core.
http2 = false;
locations."=/" = {
tryFiles = "$uri $uri/index.html =404";
root = pkgs.runCommand "index"
{
nativeBuildInputs = [ depot.tools.cheddar ];
} ''
mkdir -p $out
cheddar README.md < ${./README.md} > $out/index.html
find $out
'';
};
locations."/" = {
proxyPass = "http://unix:/run/nar-bridge.sock:/";
extraConfig = ''
# Restrict allowed HTTP methods
limit_except GET HEAD {
# nar bridge allows to upload nars via PUT
deny all;
}
# Enable proxy cache
proxy_cache nar-bridge;
proxy_cache_key "$scheme$proxy_host$request_uri";
proxy_cache_valid 200 301 302 10m; # Cache responses for 10 minutes
proxy_cache_valid 404 1m; # Cache 404 responses for 1 minute
proxy_cache_min_uses 2; # Cache only if the object is requested at least twice
proxy_cache_use_stale error timeout updating;
zstd on;
zstd_types application/x-nix-nar;
'';
};
};
# use more cores for compression
appendConfig = ''
worker_processes auto;
'';
proxyCachePath."nar-bridge" = {
enable = true;
levels = "1:2";
keysZoneName = "nar-bridge";
# Put our 1TB NVME to good use
maxSize = "200G";
inactive = "10d";
useTempPath = false;
};
};
services.nar-bridge = {
enable = true;
settings = {
blobservices = {
root = {
type = "objectstore";
object_store_url = "file:///var/lib/nar-bridge/blobs.object_store";
object_store_options = { };
};
};
directoryservices = {
root = {
type = "redb";
is_temporary = false;
path = "/var/lib/nar-bridge/directories.redb";
};
};
pathinfoservices = {
root = {
type = "cache";
near = "redb";
far = "cache-nixos-org";
};
redb = {
type = "redb";
is_temporary = false;
path = "/var/lib/nar-bridge/pathinfo.redb";
};
"cache-nixos-org" = {
type = "nix";
base_url = "https://cache.nixos.org";
blob_service = "root";
directory_service = "root";
public_keys = [
"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
];
};
};
};
};
systemd.tmpfiles.rules = [
# Put the data in the big disk
"d /tank/nar-bridge 0755 nar-bridge nar-bridge -"
# Cache responses on NVME
"d /var/cache/nginx 0755 ${config.services.nginx.user} ${config.services.nginx.group} -"
];
fileSystems."/var/lib/nar-bridge" = {
device = "/tank/nar-bridge";
options = [
"bind"
"nofail"
];
};
systemd.services.nar-bridge = {
unitConfig.RequiresMountsFor = "/var/lib/nar-bridge";
# twice the normal allowed limit, same as nix-daemon
serviceConfig.LimitNOFILE = "1048576";
};
}
|
