From e9df3608e1a6dd580d5825a7cc5972a59f0d464d Mon Sep 17 00:00:00 2001 From: sterni Date: Wed, 30 Nov 2022 15:52:08 +0100 Subject: feat(sterni/machines/edwin): automatically sync repos with github MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I primarily use GitHub for most of these preexisting repositories, but they should be properly replicated on edwin in case I want to stop. Pushing the respective refs manually is cumbersome and error prone, so let's automate it. The repositories are basically chowned to git:git currently and `git fetch 'refs/*:refs/*' --prune` is execute regularly to update the repository. In the future I could contemplate doing it the other way round – using edwin as upstream and using `git push --mirror` to update the GitHub repositories. Change-Id: Icb8a11223c0b4d3c8ce9a2da7fb2b4d4df4887f8 Reviewed-on: https://cl.tvl.fyi/c/depot/+/7486 Tested-by: BuildkiteCI Reviewed-by: sterni --- users/sterni/machines/edwin/default.nix | 2 +- .../sterni/machines/edwin/http/code.sterni.lv.nix | 107 +++++++++++++++++++-- 2 files changed, 101 insertions(+), 8 deletions(-) diff --git a/users/sterni/machines/edwin/default.nix b/users/sterni/machines/edwin/default.nix index af1dceada5..de34aa78af 100644 --- a/users/sterni/machines/edwin/default.nix +++ b/users/sterni/machines/edwin/default.nix @@ -72,7 +72,7 @@ root.openssh.authorizedKeys.keys = depot.users.sterni.keys.all; lukas = { isNormalUser = true; - extraGroups = [ "wheel" "http" ]; + extraGroups = [ "wheel" "http" "git" ]; openssh.authorizedKeys.keys = depot.users.sterni.keys.all; shell = "${pkgs.fish}/bin/fish"; }; diff --git a/users/sterni/machines/edwin/http/code.sterni.lv.nix b/users/sterni/machines/edwin/http/code.sterni.lv.nix index 8b842c9976..5e917630cf 100644 --- a/users/sterni/machines/edwin/http/code.sterni.lv.nix +++ b/users/sterni/machines/edwin/http/code.sterni.lv.nix @@ -1,6 +1,5 @@ { depot, pkgs, lib, config, ... }: -# TODO(sterni): automatically sync repositories with upstream if needed let virtualHost = "code.sterni.lv"; @@ -10,6 +9,7 @@ let repos = { spacecookie = { description = "gopher server (and library for Haskell)"; + upstream = "https://github.com/sternenseemann/spacecookie.git"; }; }; } @@ -18,9 +18,11 @@ let repos = { emoji-generic = { description = "generic emoji library for Haskell"; + upstream = "https://github.com/sternenseemann/emoji-generic.git"; }; grav2ty = { description = "“realistic” 2d space game"; + upstream = "https://github.com/sternenseemann/grav2ty.git"; }; haskell-dot-time = { description = "UTC-centric time library for haskell with dot time support"; @@ -29,6 +31,7 @@ let buchstabensuppe = { description = "toy font rendering for low pixelcount, high contrast displays"; defaultBranch = "main"; + upstream = "https://github.com/sternenseemann/buchstabensuppe.git"; }; }; } @@ -37,29 +40,31 @@ let repos = { gopher-proxy = { description = "Gopher over HTTP proxy"; + upstream = "https://github.com/sternenseemann/gopher-proxy.git"; }; likely-music = { description = "experimental application for probabilistic music composition"; + upstream = "https://github.com/sternenseemann/likely-music.git"; }; logbook = { description = "file format for keeping a personal log"; + upstream = "https://github.com/sternenseemann/logbook.git"; }; sternenblog = { description = "file based cgi blog software"; + upstream = "https://github.com/sternenseemann/sternenblog.git"; }; }; } ]; + repoPath = name: repo: repo.path or "/srv/git/${name}.git"; + cgitRepoEntry = name: repo: - let - repoName = repo.name or name; - path = repo.path or "${repoName}.git"; - in lib.concatStringsSep "\n" ( [ - "repo.url=${repoName}" - "repo.path=/srv/git/${path}" + "repo.url=${name}" + "repo.path=${repoPath name repo}" ] ++ lib.optional (repo ? description) "repo.desc=${repo.description}" ++ lib.optional (repo ? defaultBranch) "repo.defbranch=${repo.defaultBranch}" @@ -111,6 +116,25 @@ let ) repoSections } ''; + + /* Merge a list of attrs, but fail when the same attribute occurs twice. + + Type: [ attrs ] -> attrs + */ + mergeManyDistinctAttrs = lib.foldAttrs + ( + val: nul: + if nul == null then val else throw "Every attribute name may occur only once" + ) + null; + + flatRepos = mergeManyDistinctAttrs + (builtins.map (section: section.repos) repoSections); + + reposToMirror = lib.filterAttrs (_: repo: repo ? upstream) flatRepos; + + # User and group name used for running the mirror scripts + mirroredReposOwner = "git"; in { @@ -138,5 +162,74 @@ in } ''; }; + + users = { + users.${mirroredReposOwner} = { + group = mirroredReposOwner; + isSystemUser = true; + }; + + groups.${mirroredReposOwner} = { }; + }; + + + systemd.timers = lib.mapAttrs' + ( + name: repo: + { + name = "mirror-${name}"; + value = { + description = "regularly update mirror git repository ${name}"; + wantedBy = [ "timers.target" ]; + enable = true; + timerConfig = { + # Fire every 6h and distribute the workload over next 6h randomly + OnCalendar = "*-*-* 00/6:00:00"; + AccuracySec = "6h"; + RandomizedDelaySec = "6h"; + Persistent = true; + }; + }; + } + ) + reposToMirror; + + systemd.services = lib.mapAttrs' + ( + name: repo: + { + name = "mirror-${name}"; + value = { + description = "mirror git repository ${name}"; + after = [ "network.target" ]; + script = + let + path = repoPath name repo; + in + '' + set -euo pipefail + + export PATH="${lib.makeBinPath [ pkgs.coreutils pkgs.git ]}" + + if test ! -d "${path}"; then + mkdir -p "$(dirname "${path}")" + git clone --mirror "${repo.upstream}" "${path}" + exit 0 + fi + + cd "${path}" + + git fetch "${repo.upstream}" '+refs/*:refs/*' --prune + ''; + + serviceConfig = { + Type = "oneshot"; + User = mirroredReposOwner; + Group = mirroredReposOwner; + }; + }; + } + ) + reposToMirror; }; } -- cgit 1.4.1