diff options
Diffstat (limited to 'ops')
32 files changed, 1257 insertions, 59 deletions
diff --git a/ops/infra/gcp/default.tf b/ops/infra/gcp/default.tf index 2cb57836fa6d..d2e31090b560 100644 --- a/ops/infra/gcp/default.tf +++ b/ops/infra/gcp/default.tf @@ -3,6 +3,7 @@ provider "google" { project = "tazjins-infrastructure" region = "europe-north1" + version = "~> 2.20" } # Configure a storage bucket in which to keep Terraform state and @@ -26,6 +27,7 @@ resource "google_project_services" "primary" { "bigquery-json.googleapis.com", "bigquerystorage.googleapis.com", "cloudapis.googleapis.com", + "cloudbuild.googleapis.com", "clouddebugger.googleapis.com", "cloudfunctions.googleapis.com", "cloudkms.googleapis.com", @@ -34,7 +36,9 @@ resource "google_project_services" "primary" { "container.googleapis.com", "containerregistry.googleapis.com", "datastore.googleapis.com", + "distance-matrix-backend.googleapis.com", "dns.googleapis.com", + "gmail.googleapis.com", "iam.googleapis.com", "iamcredentials.googleapis.com", "logging.googleapis.com", @@ -42,6 +46,7 @@ resource "google_project_services" "primary" { "oslogin.googleapis.com", "pubsub.googleapis.com", "run.googleapis.com", + "secretmanager.googleapis.com", "servicemanagement.googleapis.com", "serviceusage.googleapis.com", "sourcerepo.googleapis.com", diff --git a/ops/infra/kubernetes/cgit/config.yaml b/ops/infra/kubernetes/cgit/config.yaml index 43bfe9d7fb45..73392adaad81 100644 --- a/ops/infra/kubernetes/cgit/config.yaml +++ b/ops/infra/kubernetes/cgit/config.yaml @@ -8,6 +8,8 @@ data: username: "Z2l0LXRhemppbi5nbWFpbC5jb20=" # This credential is a GCSR 'gitcookie' token. password: '{{ passLookup "gcsr-tazjin-password" | b64enc }}' + # This credential is an OAuth token for builds.sr.ht + sourcehut: '{{ passLookup "sr.ht-token" | b64enc }}' --- apiVersion: apps/v1 kind: Deployment @@ -16,7 +18,7 @@ metadata: labels: app: cgit spec: - replicas: 2 + replicas: 1 selector: matchLabels: app: cgit @@ -53,6 +55,11 @@ spec: secretKeyRef: name: gcsr-secrets key: password + - name: SRHT_TOKEN + valueFrom: + secretKeyRef: + name: gcsr-secrets + key: sourcehut volumeMounts: - name: git-volume mountPath: /git diff --git a/ops/infra/kubernetes/https-lb/ingress.yaml b/ops/infra/kubernetes/https-lb/ingress.yaml index 069771a4217f..930affec7a15 100644 --- a/ops/infra/kubernetes/https-lb/ingress.yaml +++ b/ops/infra/kubernetes/https-lb/ingress.yaml @@ -9,14 +9,22 @@ metadata: networking.gke.io/managed-certificates: tazj-in, git-tazj-in, www-tazj-in, oslo-pub spec: rules: - # Route blog to the blog ... + # Route website to, well, the website ... - host: tazj.in http: paths: - path: /* backend: - serviceName: tazblog - servicePort: 8000 + serviceName: website + servicePort: 8080 + # Same for www.* (the redirect is handled by the website nginx) + - host: www.tazj.in + http: + paths: + - path: /* + backend: + serviceName: website + servicePort: 8080 # Route git.tazj.in to the cgit pods - host: git.tazj.in http: diff --git a/ops/infra/kubernetes/nginx/nginx.yaml b/ops/infra/kubernetes/nginx/nginx.yaml index 983b265bafab..61678a85bca0 100644 --- a/ops/infra/kubernetes/nginx/nginx.yaml +++ b/ops/infra/kubernetes/nginx/nginx.yaml @@ -25,7 +25,7 @@ spec: config: {{ insertFile "nginx.conf" | sha1sum }} spec: containers: - - name: tazblog + - name: nginx image: nixery.local/shell/third_party.nginx:{{ .version }} command: ["/bin/bash", "-c"] args: diff --git a/ops/infra/kubernetes/nixery/known_hosts b/ops/infra/kubernetes/nixery/known_hosts index 6a2f84b5fb60..7faf21f69bf8 100644 --- a/ops/infra/kubernetes/nixery/known_hosts +++ b/ops/infra/kubernetes/nixery/known_hosts @@ -1,2 +1,3 @@ github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== 140.82.118.4 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== +[source.developers.google.com]:2022,[172.253.120.82]:2022 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB5Iy4/cq/gt/fPqe3uyMy4jwv1Alc94yVPxmnwNhBzJqEV5gRPiRk5u4/JJMbbu9QUVAguBABxL7sBZa5PH/xY= diff --git a/ops/infra/kubernetes/primary-cluster.yaml b/ops/infra/kubernetes/primary-cluster.yaml index 1d5d33e0bb26..3d601b80cd01 100644 --- a/ops/infra/kubernetes/primary-cluster.yaml +++ b/ops/infra/kubernetes/primary-cluster.yaml @@ -30,7 +30,7 @@ include: account: nixery@tazjins-infrastructure.iam.gserviceaccount.com repo: ssh://tazjin@gmail.com@source.developers.google.com:2022/p/tazjins-infrastructure/r/depot popularity: 'popularity-nixos-unstable-3140fa89c51233397f496f49014f6b23216667c2.json' - - name: tazblog + - name: website - name: cgit - name: https-lb - name: nginx diff --git a/ops/infra/kubernetes/tazblog/config.yaml b/ops/infra/kubernetes/tazblog/config.yaml deleted file mode 100644 index dc63ce8e4b38..000000000000 --- a/ops/infra/kubernetes/tazblog/config.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tazblog - labels: - app: tazblog -spec: - replicas: 2 - selector: - matchLabels: - app: tazblog - template: - metadata: - labels: - app: tazblog - spec: - containers: - - name: tazblog - image: nixery.local/shell/web.tazblog:{{ gitHEAD }} - command: [ "tazblog" ] ---- -apiVersion: v1 -kind: Service -metadata: - name: tazblog -spec: - type: NodePort - selector: - app: tazblog - ports: - - protocol: TCP - port: 8000 - targetPort: 8000 diff --git a/ops/infra/kubernetes/website/config.yaml b/ops/infra/kubernetes/website/config.yaml new file mode 100644 index 000000000000..02de735b05d0 --- /dev/null +++ b/ops/infra/kubernetes/website/config.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: website + labels: + app: website +spec: + replicas: 3 + selector: + matchLabels: + app: website + template: + metadata: + labels: + app: website + spec: + containers: + - name: website + image: nixery.local/shell/web.homepage:{{ gitHEAD }} + env: + - name: CONTAINER_SETUP + value: "true" + command: [ "homepage" ] +--- +apiVersion: v1 +kind: Service +metadata: + name: website +spec: + type: NodePort + selector: + app: website + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 diff --git a/ops/kontemplate/README.md b/ops/kontemplate/README.md index dd02943f89c5..998e61a61928 100644 --- a/ops/kontemplate/README.md +++ b/ops/kontemplate/README.md @@ -96,18 +96,18 @@ to only update the `api` resource sets and the `frontend/user-page` resource set ## Installation -It is recommended to install Kontemplate from the signed binary releases available on the -[releases page][]. Release binaries are available for Linux, OS X, FreeBSD and Windows. +It is recommended to install Kontemplate from the [Nix](https://nixos.org/) package set, +where it is available since NixOS 17.09 as `kontemplate`. -### NixOS +If using Nix is not an option for you, several other methods of installation are +available: -Kontemplate has been included in [NixOS](https://nixos.org/) since version 17.09. +### Binary releases -It is available as `kontemplate` from the default Nix package set. +Signed binary releases are available on the [releases page][] for Linux, OS X, FreeBSD and +Windows. -### Arch Linux - -An [AUR package][] is available for Arch Linux and other `pacman`-based distributions. +Releases are signed with the GPG key `DCF34CFAC1AC44B87E26333136EE34814F6D294A`. ### Building from source @@ -184,4 +184,3 @@ Please follow the [code of conduct](CODE_OF_CONDUCT.md). [Kontemplate]: http://kontemplate.works [Helm]: https://helm.sh/ [releases page]: https://github.com/tazjin/kontemplate/releases -[AUR package]: https://aur.archlinux.org/packages/kontemplate-git/ diff --git a/ops/mq_cli/.gitignore b/ops/mq_cli/.gitignore new file mode 100644 index 000000000000..5bd19a47c3b3 --- /dev/null +++ b/ops/mq_cli/.gitignore @@ -0,0 +1,4 @@ +/target/ +**/*.rs.bk +.idea/ +*.iml diff --git a/ops/mq_cli/CODE_OF_CONDUCT.md b/ops/mq_cli/CODE_OF_CONDUCT.md new file mode 100644 index 000000000000..c4013ac13ebc --- /dev/null +++ b/ops/mq_cli/CODE_OF_CONDUCT.md @@ -0,0 +1,20 @@ +A SERMON ON ETHICS AND LOVE +=========================== + +One day Mal-2 asked the messenger spirit Saint Gulik to approach the Goddess and request Her presence for some desperate advice. Shortly afterwards the radio came on by itself, and an ethereal female Voice said **YES?** + +"O! Eris! Blessed Mother of Man! Queen of Chaos! Daughter of Discord! Concubine of Confusion! O! Exquisite Lady, I beseech You to lift a heavy burden from my heart!" + +**WHAT BOTHERS YOU, MAL? YOU DON'T SOUND WELL.** + +"I am filled with fear and tormented with terrible visions of pain. Everywhere people are hurting one another, the planet is rampant with injustices, whole societies plunder groups of their own people, mothers imprison sons, children perish while brothers war. O, woe." + +**WHAT IS THE MATTER WITH THAT, IF IT IS WHAT YOU WANT TO DO?** + +"But nobody Wants it! Everybody hates it." + +**OH. WELL, THEN *STOP*.** + +At which moment She turned herself into an aspirin commercial and left The Polyfather stranded alone with his species. + +SINISTER DEXTER HAS A BROKEN SPIROMETER. diff --git a/ops/mq_cli/Cargo.lock b/ops/mq_cli/Cargo.lock new file mode 100644 index 000000000000..f418d77c34ff --- /dev/null +++ b/ops/mq_cli/Cargo.lock @@ -0,0 +1,159 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cc" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "clap" +version = "2.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hermit-abi" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libc" +version = "0.2.66" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "mq" +version = "1.0.0" +dependencies = [ + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)", + "posix_mq 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nix" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "posix_mq" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "vec_map" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" +"checksum hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eff2656d88f158ce120947499e971d743c05dbcbed62e5bd2f38f1698bbc3772" +"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +"checksum nix 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dd0eaf8df8bab402257e0a5c17a254e4cc1f72a93588a1ddfb5d356c801aa7cb" +"checksum posix_mq 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "13ae339e13cc96902a4597a5aab6b76473093969c55d36ba33f6a7bf3268573f" +"checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +"checksum unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/ops/mq_cli/Cargo.toml b/ops/mq_cli/Cargo.toml new file mode 100644 index 000000000000..b412d88787fa --- /dev/null +++ b/ops/mq_cli/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mq" +version = "1.0.0" +authors = ["Vincent Ambo <mail@tazj.in>"] + +[dependencies] +clap = "2.33" +libc = "0.2" +nix = "0.16" +posix_mq = "0.9" diff --git a/ops/mq_cli/LICENSE b/ops/mq_cli/LICENSE new file mode 100644 index 000000000000..e289cbab81e0 --- /dev/null +++ b/ops/mq_cli/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2017-2018 Langler AS +Copyright (c) 2019-2020 Vincent Ambo + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ops/mq_cli/README.md b/ops/mq_cli/README.md new file mode 100644 index 000000000000..e612553e7447 --- /dev/null +++ b/ops/mq_cli/README.md @@ -0,0 +1,31 @@ +mq-cli +====== + +This project provides a very simple CLI interface to [POSIX message queues][]. + +It can be used to create and inspect queues, as well as send and +receive messages from them. + +``` +1.0.0 +Administrate and inspect POSIX message queues + +USAGE: + mq <SUBCOMMAND> + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information + +SUBCOMMANDS: + create Create a new queue + help Prints this message or the help of the given subcommand(s) + inspect inspect details about a queue + ls list message queues + receive Receive a message from a queue + rlimit Get the message queue rlimit + send Send a message to a queue +``` + + +[POSIX message queues]: https://linux.die.net/man/7/mq_overview diff --git a/ops/mq_cli/default.nix b/ops/mq_cli/default.nix new file mode 100644 index 000000000000..190a05c80577 --- /dev/null +++ b/ops/mq_cli/default.nix @@ -0,0 +1,3 @@ +{ pkgs, ... }: + +pkgs.third_party.naersk.buildPackage ./. diff --git a/ops/mq_cli/src/main.rs b/ops/mq_cli/src/main.rs new file mode 100644 index 000000000000..55ff0064295d --- /dev/null +++ b/ops/mq_cli/src/main.rs @@ -0,0 +1,223 @@ +extern crate clap; +extern crate posix_mq; +extern crate libc; +extern crate nix; + +use clap::{App, SubCommand, Arg, ArgMatches, AppSettings}; +use posix_mq::{Name, Queue, Message}; +use std::fs::{read_dir, File}; +use std::io::{self, Read, Write}; +use std::process::exit; + +fn run_ls() { + let mqueues = read_dir("/dev/mqueue") + .expect("Could not read message queues"); + + for queue in mqueues { + let path = queue.unwrap().path(); + let status = { + let mut file = File::open(&path) + .expect("Could not open queue file"); + + let mut content = String::new(); + file.read_to_string(&mut content).expect("Could not read queue file"); + + content + }; + + let queue_name = path.components().last().unwrap() + .as_os_str() + .to_string_lossy(); + + println!("/{}: {}", queue_name, status) + }; +} + +fn run_inspect(queue_name: &str) { + let name = Name::new(queue_name).expect("Invalid queue name"); + let queue = Queue::open(name).expect("Could not open queue"); + + println!("Queue {}:\n", queue_name); + println!("Max. message size: {} bytes", queue.max_size()); + println!("Max. # of pending messages: {}", queue.max_pending()); +} + +fn run_create(cmd: &ArgMatches) { + if let Some(rlimit) = cmd.value_of("rlimit") { + set_rlimit(rlimit.parse().expect("Invalid rlimit value")); + } + + let name = Name::new(cmd.value_of("queue").unwrap()) + .expect("Invalid queue name"); + + let max_pending: i64 = cmd.value_of("max-pending").unwrap().parse().unwrap(); + let max_size: i64 = cmd.value_of("max-size").unwrap().parse().unwrap(); + + let queue = Queue::create(name, max_pending, max_size * 1024); + + match queue { + Ok(_) => println!("Queue created successfully"), + Err(e) => { + writeln!(io::stderr(), "Could not create queue: {}", e).ok(); + exit(1); + }, + }; +} + +fn run_receive(queue_name: &str) { + let name = Name::new(queue_name).expect("Invalid queue name"); + let queue = Queue::open(name).expect("Could not open queue"); + + let message = match queue.receive() { + Ok(msg) => msg, + Err(e) => { + writeln!(io::stderr(), "Failed to receive message: {}", e).ok(); + exit(1); + } + }; + + // Attempt to write the message out as a string, but write out raw bytes if it turns out to not + // be UTF-8 encoded data. + match String::from_utf8(message.data.clone()) { + Ok(string) => println!("{}", string), + Err(_) => { + writeln!(io::stderr(), "Message not UTF-8 encoded!").ok(); + io::stdout().write(message.data.as_ref()).ok(); + } + }; +} + +fn run_send(queue_name: &str, content: &str) { + let name = Name::new(queue_name).expect("Invalid queue name"); + let queue = Queue::open(name).expect("Could not open queue"); + + let message = Message { + data: content.as_bytes().to_vec(), + priority: 0, + }; + + match queue.send(&message) { + Ok(_) => (), + Err(e) => { + writeln!(io::stderr(), "Could not send message: {}", e).ok(); + exit(1); + } + } +} + +fn run_rlimit() { + let mut rlimit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + + let mut errno = 0; + unsafe { + let res = libc::getrlimit(libc::RLIMIT_MSGQUEUE, &mut rlimit); + if res != 0 { + errno = nix::errno::errno(); + } + }; + + if errno != 0 { + writeln!(io::stderr(), "Could not get message queue rlimit: {}", errno).ok(); + } else { + println!("Message queue rlimit:"); + println!("Current limit: {}", rlimit.rlim_cur); + println!("Maximum limit: {}", rlimit.rlim_max); + } +} + +fn set_rlimit(new_limit: u64) { + let rlimit = libc::rlimit { + rlim_cur: new_limit, + rlim_max: new_limit, + }; + + let mut errno: i32 = 0; + unsafe { + let res = libc::setrlimit(libc::RLIMIT_MSGQUEUE, &rlimit); + if res != 0 { + errno = nix::errno::errno(); + } + } + + match errno { + 0 => println!("Set RLIMIT_MSGQUEUE hard limit to {}", new_limit), + _ => { + // Not mapping these error codes to messages for now, the user can + // look up the meaning in setrlimit(2). + panic!("Could not set hard limit: {}", errno); + } + }; +} + +fn main() { + let ls = SubCommand::with_name("ls").about("list message queues"); + + let queue_arg = Arg::with_name("queue").required(true).takes_value(true); + + let rlimit_arg = Arg::with_name("rlimit") + .help("RLIMIT_MSGQUEUE to set for this command") + .long("rlimit") + .takes_value(true); + + let inspect = SubCommand::with_name("inspect") + .about("inspect details about a queue") + .arg(&queue_arg); + + let create = SubCommand::with_name("create") + .about("Create a new queue") + .arg(&queue_arg) + .arg(&rlimit_arg) + .arg(Arg::with_name("max-size") + .help("maximum message size (in kB)") + .long("max-size") + .required(true) + .takes_value(true)) + .arg(Arg::with_name("max-pending") + .help("maximum # of pending messages") + .long("max-pending") + .required(true) + .takes_value(true)); + + let receive = SubCommand::with_name("receive") + .about("Receive a message from a queue") + .arg(&queue_arg); + + let send = SubCommand::with_name("send") + .about("Send a message to a queue") + .arg(&queue_arg) + .arg(Arg::with_name("message") + .help("the message to send") + .required(true)); + + let rlimit = SubCommand::with_name("rlimit") + .about("Get the message queue rlimit") + .setting(AppSettings::SubcommandRequiredElseHelp); + + let matches = App::new("mq") + .setting(AppSettings::SubcommandRequiredElseHelp) + .version("1.0.0") + .about("Administrate and inspect POSIX message queues") + .subcommand(ls) + .subcommand(inspect) + .subcommand(create) + .subcommand(receive) + .subcommand(send) + .subcommand(rlimit) + .get_matches(); + + match matches.subcommand() { + ("ls", _) => run_ls(), + ("inspect", Some(cmd)) => run_inspect(cmd.value_of("queue").unwrap()), + ("create", Some(cmd)) => run_create(cmd), + ("receive", Some(cmd)) => run_receive(cmd.value_of("queue").unwrap()), + ("send", Some(cmd)) => run_send( + cmd.value_of("queue").unwrap(), + cmd.value_of("message").unwrap() + ), + ("rlimit", _) => run_rlimit(), + _ => unimplemented!(), + } +} diff --git a/ops/nixos/nugget/default.nix b/ops/nixos/nugget/default.nix index 6a68926e43da..3473968706d3 100644 --- a/ops/nixos/nugget/default.nix +++ b/ops/nixos/nugget/default.nix @@ -14,6 +14,7 @@ in pkgs.lib.fix(self: { hardware = { pulseaudio.enable = true; cpu.intel.updateMicrocode = true; + u2f.enable = true; }; boot = { @@ -55,7 +56,16 @@ in pkgs.lib.fix(self: { ]; # Open Chromecast-related ports & servedir + firewall.enable = false; firewall.allowedTCPPorts = [ 4242 5556 5558 ]; + + # Connect to the WiFi to let the Chromecast work. + wireless.enable = true; + wireless.networks = { + "How do I computer?" = { + psk = "washyourface"; + }; + }; }; # Generate an immutable /etc/resolv.conf from the nameserver settings @@ -75,6 +85,7 @@ in pkgs.lib.fix(self: { lieer ops.kontemplate third_party.git + third_party.guile tools.emacs ]) ++ @@ -82,6 +93,7 @@ in pkgs.lib.fix(self: { (with nixpkgs; [ age bat + cachix chromium curl direnv @@ -90,11 +102,15 @@ in pkgs.lib.fix(self: { fd gnupg go + google-chrome google-cloud-sdk htop + i3lock imagemagick jq + keybase-gui kubectl + miller msmtp nix-prefetch-github notmuch @@ -112,8 +128,11 @@ in pkgs.lib.fix(self: { spotify tokei tree + unzip vlc xclip + yubico-piv-tool + yubikey-personalization ]); fileSystems = { @@ -138,10 +157,21 @@ in pkgs.lib.fix(self: { fonts = { fonts = with nixpkgs; [ corefonts + dejavu_fonts input-fonts + jetbrains-mono noto-fonts-cjk noto-fonts-emoji ]; + + fontconfig = { + hinting.enable = true; + subpixel.lcdfilter = "light"; + + defaultFonts = { + monospace = [ "JetBrains Mono" ]; + }; + }; }; # Configure location (Vauxhall, London) for services that need it. @@ -154,6 +184,13 @@ in pkgs.lib.fix(self: { services.redshift.enable = true; services.openssh.enable = true; + services.keybase.enable = true; + + # Required for Yubikey usage as smartcard + services.pcscd.enable = true; + services.udev.packages = [ + nixpkgs.yubikey-personalization + ]; services.xserver = { enable = true; diff --git a/ops/posix_mq.rs/.gitignore b/ops/posix_mq.rs/.gitignore new file mode 100644 index 000000000000..e5b6fdb28e32 --- /dev/null +++ b/ops/posix_mq.rs/.gitignore @@ -0,0 +1,3 @@ +/target/ +**/*.rs.bk +.idea/ diff --git a/ops/posix_mq.rs/CODE_OF_CONDUCT.md b/ops/posix_mq.rs/CODE_OF_CONDUCT.md new file mode 100644 index 000000000000..c4013ac13ebc --- /dev/null +++ b/ops/posix_mq.rs/CODE_OF_CONDUCT.md @@ -0,0 +1,20 @@ +A SERMON ON ETHICS AND LOVE +=========================== + +One day Mal-2 asked the messenger spirit Saint Gulik to approach the Goddess and request Her presence for some desperate advice. Shortly afterwards the radio came on by itself, and an ethereal female Voice said **YES?** + +"O! Eris! Blessed Mother of Man! Queen of Chaos! Daughter of Discord! Concubine of Confusion! O! Exquisite Lady, I beseech You to lift a heavy burden from my heart!" + +**WHAT BOTHERS YOU, MAL? YOU DON'T SOUND WELL.** + +"I am filled with fear and tormented with terrible visions of pain. Everywhere people are hurting one another, the planet is rampant with injustices, whole societies plunder groups of their own people, mothers imprison sons, children perish while brothers war. O, woe." + +**WHAT IS THE MATTER WITH THAT, IF IT IS WHAT YOU WANT TO DO?** + +"But nobody Wants it! Everybody hates it." + +**OH. WELL, THEN *STOP*.** + +At which moment She turned herself into an aspirin commercial and left The Polyfather stranded alone with his species. + +SINISTER DEXTER HAS A BROKEN SPIROMETER. diff --git a/ops/posix_mq.rs/Cargo.lock b/ops/posix_mq.rs/Cargo.lock new file mode 100644 index 000000000000..fdd0086c4dd3 --- /dev/null +++ b/ops/posix_mq.rs/Cargo.lock @@ -0,0 +1,54 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cc" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.66" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "nix" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "posix_mq" +version = "0.9.0" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +"checksum nix 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dd0eaf8df8bab402257e0a5c17a254e4cc1f72a93588a1ddfb5d356c801aa7cb" +"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" diff --git a/ops/posix_mq.rs/Cargo.toml b/ops/posix_mq.rs/Cargo.toml new file mode 100644 index 000000000000..d72e87a3dcef --- /dev/null +++ b/ops/posix_mq.rs/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "posix_mq" +version = "0.9.0" +authors = ["Vincent Ambo <mail@tazj.in>"] +description = "(Higher-level) Rust bindings to POSIX message queues" +license = "MIT" +repository = "https://git.tazj.in/tree/ops/posix_mq.rs" + +[dependencies] +nix = "0.16" +libc = "0.2" diff --git a/ops/posix_mq.rs/LICENSE b/ops/posix_mq.rs/LICENSE new file mode 100644 index 000000000000..2389546b1383 --- /dev/null +++ b/ops/posix_mq.rs/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017-2020 Vincent Ambo + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ops/posix_mq.rs/README.md b/ops/posix_mq.rs/README.md new file mode 100644 index 000000000000..9370c6c08740 --- /dev/null +++ b/ops/posix_mq.rs/README.md @@ -0,0 +1,33 @@ +posix_mq +======== + +[](https://travis-ci.org/aprilabank/posix_mq.rs) +[](https://crates.io/crates/posix_mq) + +This is a simple, relatively high-level library for the POSIX [message queue API][]. It wraps the lower-level API in a +simpler interface with more robust error handling. + +Check out this project's [sister library][] in Kotlin. + +Usage example: + +```rust +// Values that need to undergo validation are wrapped in safe types: +let name = Name::new("/test-queue").unwrap(); + +// Queue creation with system defaults is simple: +let queue = Queue::open_or_create(name).expect("Opening queue failed"); + +// Sending a message: +let message = Message { + data: "test-message".as_bytes().to_vec(), + priority: 0, +}; +queue.send(&message).expect("message sending failed"); + +// ... and receiving it! +let result = queue.receive().expect("message receiving failed"); +``` + +[message queue API]: https://linux.die.net/man/7/mq_overview +[sister library]: https://github.com/aprilabank/posix_mq.kt diff --git a/ops/posix_mq.rs/default.nix b/ops/posix_mq.rs/default.nix new file mode 100644 index 000000000000..190a05c80577 --- /dev/null +++ b/ops/posix_mq.rs/default.nix @@ -0,0 +1,3 @@ +{ pkgs, ... }: + +pkgs.third_party.naersk.buildPackage ./. diff --git a/ops/posix_mq.rs/src/error.rs b/ops/posix_mq.rs/src/error.rs new file mode 100644 index 000000000000..1ef585c01efb --- /dev/null +++ b/ops/posix_mq.rs/src/error.rs @@ -0,0 +1,130 @@ +use nix; +use std::error; +use std::fmt; +use std::io; +use std::num; + +/// This module implements a simple error type to match the errors that can be thrown from the C +/// functions as well as some extra errors resulting from internal validations. +/// +/// As this crate exposes an opinionated API to the POSIX queues certain errors have been +/// ignored: +/// +/// * ETIMEDOUT: The low-level timed functions are not exported and this error can not occur. +/// * EAGAIN: Non-blocking queue calls are not supported. +/// * EINVAL: Same reason as ETIMEDOUT +/// * EMSGSIZE: The message size is immutable after queue creation and this crate checks it. +/// * ENAMETOOLONG: This crate performs name validation +/// +/// If an unexpected error is encountered it will be wrapped appropriately and should be reported +/// as a bug on https://github.com/aprilabank/posix_mq.rs + +#[derive(Debug)] +pub enum Error { + // These errors are raised inside of the library + InvalidQueueName(&'static str), + ValueReadingError(io::Error), + MessageSizeExceeded(), + MaximumMessageSizeExceeded(), + MaximumMessageCountExceeded(), + + // These errors match what is described in the man pages (from mq_overview(7) onwards). + PermissionDenied(), + InvalidQueueDescriptor(), + QueueCallInterrupted(), + QueueAlreadyExists(), + QueueNotFound(), + InsufficientMemory(), + InsufficientSpace(), + + // These two are (hopefully) unlikely in modern systems + ProcessFileDescriptorLimitReached(), + SystemFileDescriptorLimitReached(), + + // If an unhandled / unknown / unexpected error occurs this error will be used. + // In those cases bug reports would be welcome! + UnknownForeignError(nix::errno::Errno), + + // Some other unexpected / unknown error occured. This is probably an error from + // the nix crate. Bug reports also welcome for this! + UnknownInternalError(Option<nix::Error>), +} + +impl error::Error for Error { + fn description(&self) -> &str { + use Error::*; + match *self { + // This error contains more sensible description strings already + InvalidQueueName(e) => e, + ValueReadingError(_) => "error reading system configuration for message queues", + MessageSizeExceeded() => "message is larger than maximum size for specified queue", + MaximumMessageSizeExceeded() => "specified queue message size exceeds system maximum", + MaximumMessageCountExceeded() => "specified queue message count exceeds system maximum", + PermissionDenied() => "permission to the specified queue was denied", + InvalidQueueDescriptor() => "the internal queue descriptor was invalid", + QueueCallInterrupted() => "queue method interrupted by signal", + QueueAlreadyExists() => "the specified queue already exists", + QueueNotFound() => "the specified queue could not be found", + InsufficientMemory() => "insufficient memory to call queue method", + InsufficientSpace() => "insufficient space to call queue method", + ProcessFileDescriptorLimitReached() => + "maximum number of process file descriptors reached", + SystemFileDescriptorLimitReached() => + "maximum number of system file descriptors reached", + UnknownForeignError(_) => "unknown foreign error occured: please report a bug!", + UnknownInternalError(_) => "unknown internal error occured: please report a bug!", + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Explicitly import this to gain access to Error::description() + use std::error::Error; + f.write_str(self.description()) + } +} + +/// This from implementation is used to translate errors from the lower-level +/// C-calls into sensible Rust errors. +impl From<nix::Error> for Error { + fn from(e: nix::Error) -> Self { + match e { + nix::Error::Sys(e) => match_errno(e), + _ => Error::UnknownInternalError(Some(e)), + } + } +} + +// This implementation is used when reading system queue settings. +impl From<io::Error> for Error { + fn from(e: io::Error) -> Self { + Error::ValueReadingError(e) + } +} + +// This implementation is used when parsing system queue settings. The unknown error is returned +// here because the system is probably seriously broken if those files don't contain numbers. +impl From<num::ParseIntError> for Error { + fn from(_: num::ParseIntError) -> Self { + Error::UnknownInternalError(None) + } +} + + +fn match_errno(err: nix::errno::Errno) -> Error { + use nix::errno::Errno::*; + + match err { + EACCES => Error::PermissionDenied(), + EBADF => Error::InvalidQueueDescriptor(), + EINTR => Error::QueueCallInterrupted(), + EEXIST => Error::QueueAlreadyExists(), + EMFILE => Error::ProcessFileDescriptorLimitReached(), + ENFILE => Error::SystemFileDescriptorLimitReached(), + ENOENT => Error::QueueNotFound(), + ENOMEM => Error::InsufficientMemory(), + ENOSPC => Error::InsufficientSpace(), + _ => Error::UnknownForeignError(err), + } +} diff --git a/ops/posix_mq.rs/src/lib.rs b/ops/posix_mq.rs/src/lib.rs new file mode 100644 index 000000000000..057601eccfbd --- /dev/null +++ b/ops/posix_mq.rs/src/lib.rs @@ -0,0 +1,269 @@ +extern crate nix; +extern crate libc; + +use error::Error; +use libc::mqd_t; +use nix::mqueue; +use nix::sys::stat; +use std::ffi::CString; +use std::fs::File; +use std::io::Read; +use std::string::ToString; +use std::ops::Drop; + +pub mod error; + +#[cfg(test)] +mod tests; + +/// Wrapper type for queue names that performs basic validation of queue names before calling +/// out to C code. +#[derive(Debug, Clone, PartialEq)] +pub struct Name(CString); + +impl Name { + pub fn new<S: ToString>(s: S) -> Result<Self, Error> { + let string = s.to_string(); + + if !string.starts_with('/') { + return Err(Error::InvalidQueueName("Queue name must start with '/'")); + } + + // The C library has a special error return for this case, so I assume people must actually + // have tried just using '/' as a queue name. + if string.len() == 1 { + return Err(Error::InvalidQueueName( + "Queue name must be a slash followed by one or more characters" + )); + } + + if string.len() > 255 { + return Err(Error::InvalidQueueName("Queue name must not exceed 255 characters")); + } + + if string.matches('/').count() > 1 { + return Err(Error::InvalidQueueName("Queue name can not contain more than one slash")); + } + + // TODO: What error is being thrown away here? Is it possible? + Ok(Name(CString::new(string).unwrap())) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Message { + pub data: Vec<u8>, + pub priority: u32, +} + +/// Represents an open queue descriptor to a POSIX message queue. This carries information +/// about the queue's limitations (i.e. maximum message size and maximum message count). +#[derive(Debug)] +pub struct Queue { + name: Name, + + /// Internal file/queue descriptor. + queue_descriptor: mqd_t, + + /// Maximum number of pending messages in this queue. + max_pending: i64, + + /// Maximum size of this queue. + max_size: usize, +} + +impl Queue { + /// Creates a new queue and fails if it already exists. + /// By default the queue will be read/writable by the current user with no access for other + /// users. + /// Linux users can change this setting themselves by modifying the queue file in /dev/mqueue. + pub fn create(name: Name, max_pending: i64, max_size: i64) -> Result<Queue, Error> { + if max_pending > read_i64_from_file(MSG_MAX)? { + return Err(Error::MaximumMessageCountExceeded()); + } + + if max_size > read_i64_from_file(MSGSIZE_MAX)? { + return Err(Error::MaximumMessageSizeExceeded()); + } + + let oflags = { + let mut flags = mqueue::MQ_OFlag::empty(); + // Put queue in r/w mode + flags.toggle(mqueue::MQ_OFlag::O_RDWR); + // Enable queue creation + flags.toggle(mqueue::MQ_OFlag::O_CREAT); + // Fail if queue exists already + flags.toggle(mqueue::MQ_OFlag::O_EXCL); + flags + }; + + let attr = mqueue::MqAttr::new( + 0, max_pending, max_size, 0 + ); + + let queue_descriptor = mqueue::mq_open( + &name.0, + oflags, + default_mode(), + Some(&attr), + )?; + + Ok(Queue { + name, + queue_descriptor, + max_pending, + max_size: max_size as usize, + }) + } + + /// Opens an existing queue. + pub fn open(name: Name) -> Result<Queue, Error> { + // No extra flags need to be constructed as the default is to open and fail if the + // queue does not exist yet - which is what we want here. + let oflags = mqueue::MQ_OFlag::O_RDWR; + let queue_descriptor = mqueue::mq_open( + &name.0, + oflags, + default_mode(), + None, + )?; + + let attr = mq_getattr(queue_descriptor)?; + + Ok(Queue { + name, + queue_descriptor, + max_pending: attr.mq_maxmsg, + max_size: attr.mq_msgsize as usize, + }) + } + + /// Opens an existing queue or creates a new queue with the OS default settings. + pub fn open_or_create(name: Name) -> Result<Queue, Error> { + let oflags = { + let mut flags = mqueue::MQ_OFlag::empty(); + // Put queue in r/w mode + flags.toggle(mqueue::MQ_OFlag::O_RDWR); + // Enable queue creation + flags.toggle(mqueue::MQ_OFlag::O_CREAT); + flags + }; + + let default_pending = read_i64_from_file(MSG_DEFAULT)?; + let default_size = read_i64_from_file(MSGSIZE_DEFAULT)?; + let attr = mqueue::MqAttr::new( + 0, default_pending, default_size, 0 + ); + + let queue_descriptor = mqueue::mq_open( + &name.0, + oflags, + default_mode(), + Some(&attr), + )?; + + let actual_attr = mq_getattr(queue_descriptor)?; + + Ok(Queue { + name, + queue_descriptor, + max_pending: actual_attr.mq_maxmsg, + max_size: actual_attr.mq_msgsize as usize, + }) + } + + /// Delete a message queue from the system. This method will make the queue unavailable for + /// other processes after their current queue descriptors have been closed. + pub fn delete(self) -> Result<(), Error> { + mqueue::mq_unlink(&self.name.0)?; + drop(self); + Ok(()) + } + + /// Send a message to the message queue. + /// If the queue is full this call will block until a message has been consumed. + pub fn send(&self, msg: &Message) -> Result<(), Error> { + if msg.data.len() > self.max_size as usize { + return Err(Error::MessageSizeExceeded()); + } + + mqueue::mq_send( + self.queue_descriptor, + msg.data.as_ref(), + msg.priority, + ).map_err(|e| e.into()) + } + + /// Receive a message from the message queue. + /// If the queue is empty this call will block until a message arrives. + pub fn receive(&self) -> Result<Message, Error> { + let mut data: Vec<u8> = vec![0; self.max_size as usize]; + let mut priority: u32 = 0; + + let msg_size = mqueue::mq_receive( + self.queue_descriptor, + data.as_mut(), + &mut priority, + )?; + + data.truncate(msg_size); + Ok(Message { data, priority }) + } + + pub fn max_pending(&self) -> i64 { + self.max_pending + } + + pub fn max_size(&self) -> usize { + self.max_size + } +} + +impl Drop for Queue { + fn drop(&mut self) { + // Attempt to close the queue descriptor and discard any possible errors. + // The only error thrown in the C-code is EINVAL, which would mean that the + // descriptor has already been closed. + mqueue::mq_close(self.queue_descriptor).ok(); + } +} + +// Creates the default queue mode (0600). +fn default_mode() -> stat::Mode { + let mut mode = stat::Mode::empty(); + mode.toggle(stat::Mode::S_IRUSR); + mode.toggle(stat::Mode::S_IWUSR); + mode +} + +/// This file defines the default number of maximum pending messages in a queue. +const MSG_DEFAULT: &'static str = "/proc/sys/fs/mqueue/msg_default"; + +/// This file defines the system maximum number of pending messages in a queue. +const MSG_MAX: &'static str = "/proc/sys/fs/mqueue/msg_max"; + +/// This file defines the default maximum size of messages in a queue. +const MSGSIZE_DEFAULT: &'static str = "/proc/sys/fs/mqueue/msgsize_default"; + +/// This file defines the system maximum size for messages in a queue. +const MSGSIZE_MAX: &'static str = "/proc/sys/fs/mqueue/msgsize_max"; + +/// This method is used in combination with the above constants to find system limits. +fn read_i64_from_file(name: &str) -> Result<i64, Error> { + let mut file = File::open(name.to_string())?; + let mut content = String::new(); + file.read_to_string(&mut content)?; + Ok(content.trim().parse()?) +} + +/// The mq_getattr implementation in the nix crate hides the maximum message size and count, which +/// is very impractical. +/// To work around it, this method calls the C-function directly. +fn mq_getattr(mqd: mqd_t) -> Result<libc::mq_attr, Error> { + use std::mem; + let mut attr = unsafe { mem::uninitialized::<libc::mq_attr>() }; + let res = unsafe { libc::mq_getattr(mqd, &mut attr) }; + nix::errno::Errno::result(res) + .map(|_| attr) + .map_err(|e| e.into()) +} diff --git a/ops/posix_mq.rs/src/tests.rs b/ops/posix_mq.rs/src/tests.rs new file mode 100644 index 000000000000..7a08876aeacd --- /dev/null +++ b/ops/posix_mq.rs/src/tests.rs @@ -0,0 +1,22 @@ +use super::*; + +#[test] +fn test_open_delete() { + // Simple test with default queue settings + let name = Name::new("/test-queue").unwrap(); + let queue = Queue::open_or_create(name) + .expect("Opening queue failed"); + + let message = Message { + data: "test-message".as_bytes().to_vec(), + priority: 0, + }; + + queue.send(&message).expect("message sending failed"); + + let result = queue.receive().expect("message receiving failed"); + + assert_eq!(message, result); + + queue.delete().expect("deleting queue failed"); +} diff --git a/ops/secrets/sr.ht-token b/ops/secrets/sr.ht-token new file mode 100644 index 000000000000..53eb0d16b0e1 --- /dev/null +++ b/ops/secrets/sr.ht-token Binary files differdiff --git a/ops/sync-gcsr/default.nix b/ops/sync-gcsr/default.nix index 114ff221bed1..ae88b34124dc 100644 --- a/ops/sync-gcsr/default.nix +++ b/ops/sync-gcsr/default.nix @@ -7,4 +7,8 @@ pkgs.buildGo.program { deps = with pkgs.third_party; map (p: p.gopkg) [ gopkgs."gopkg.in".src-d.go-git ]; + + x_defs = { + "main.BuildManifest" = "${./manifest.yaml}"; + }; } diff --git a/ops/sync-gcsr/main.go b/ops/sync-gcsr/main.go index a02f6d552730..62c24a92cef0 100644 --- a/ops/sync-gcsr/main.go +++ b/ops/sync-gcsr/main.go @@ -3,19 +3,37 @@ // // sync-gcsr implements a small utility that periodically mirrors a // remote Google Cloud Source Repository to a local file path. +// +// This utility is also responsible for triggering depot builds on +// builds.sr.ht if a change is detected on the master branch. package main import ( + "encoding/json" "fmt" + "io/ioutil" "log" + "net/http" "os" "time" + "bytes" git "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/transport/http" + githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http" ) +// Path to the build manifest, added by Nix at compile time. +var BuildManifest string + +// Represents a builds.sr.ht build object as described on +// https://man.sr.ht/builds.sr.ht/api.md +type Build struct { + Manifest string `json:"manifest"` + Note string `json:"note"` + Tags []string `json:"tags"` +} + func EnvOr(key, def string) string { v := os.Getenv(key) if v == "" { @@ -25,8 +43,51 @@ func EnvOr(key, def string) string { return v } +// Trigger a build of master on builds.sr.ht +func triggerBuild(commit string) { + manifest, err := ioutil.ReadFile(BuildManifest) + if err != nil { + log.Fatalln("[ERROR] failed to read sr.ht build manifest:", err) + } + + build := Build{ + Manifest: string(manifest), + Note: fmt.Sprintf("Build of 'master' at '%s'", commit), + Tags: []string{ + "depot", "master", + }, + } + + body, _ := json.Marshal(build) + reader := ioutil.NopCloser(bytes.NewReader(body)) + + req, err := http.NewRequest("POST", "https://builds.sr.ht/api/jobs", reader) + if err != nil { + log.Fatalln("[ERROR] failed to create an HTTP request:", err) + } + + req.Header.Add("Authorization", fmt.Sprintf("token %s", os.Getenv("SRHT_TOKEN"))) + req.Header.Add("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + // This might indicate a temporary error on the SourceHut side, do + // not fail the whole program. + log.Println("failed to send builds.sr.ht request:", err) + return + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + respBody, err := ioutil.ReadAll(resp.Body) + log.Printf("received non-success response from builds.sr.ht: %s (%v)[%s]", respBody, resp.Status, err) + } else { + log.Println("triggered builds.sr.ht job for commit", commit) + } +} + // ensure that all remote branches exist locally & are up to date. -func updateBranches(auth *http.BasicAuth, repo *git.Repository) error { +func updateBranches(auth *githttp.BasicAuth, repo *git.Repository) error { origin, err := repo.Remote("origin") if err != nil { return err @@ -44,22 +105,34 @@ func updateBranches(auth *http.BasicAuth, repo *git.Repository) error { continue } - branch := plumbing.NewHashReference( - plumbing.NewBranchReferenceName(ref.Name().Short()), - ref.Hash(), - ) + name := plumbing.NewBranchReferenceName(ref.Name().Short()) + + if current, err := repo.Storer.Reference(name); err == nil { + // Determine whether the reference has changed to skip + // unnecessary modifications. + if current.Hash() == ref.Hash() { + continue + } + } + + branch := plumbing.NewHashReference(name, ref.Hash()) err := repo.Storer.SetReference(branch) if err != nil { return err } + + if ref.Name().Short() == "master" { + go triggerBuild(ref.Hash().String()) + } + log.Println("Updated branch", ref.Name().String()) } return nil } -func updateRepo(auth *http.BasicAuth, repo *git.Repository, opts *git.FetchOptions) error { +func updateRepo(auth *githttp.BasicAuth, repo *git.Repository, opts *git.FetchOptions) error { err := repo.Fetch(opts) if err == git.NoErrAlreadyUpToDate { @@ -73,7 +146,7 @@ func updateRepo(auth *http.BasicAuth, repo *git.Repository, opts *git.FetchOptio return updateBranches(auth, repo) } -func cloneRepo(dest, project, repo string, auth *http.BasicAuth) (*git.Repository, error) { +func cloneRepo(dest, project, repo string, auth *githttp.BasicAuth) (*git.Repository, error) { var cloneOpts = git.CloneOptions{ Auth: auth, URL: fmt.Sprintf("https://source.developers.google.com/p/%s/r/%s", project, repo), @@ -97,9 +170,9 @@ func main() { log.Printf("Syncing repository '%s/%s' to destination '%s'", project, repo, dest) - var auth *http.BasicAuth + var auth *githttp.BasicAuth if user != "" && pass != "" { - auth = &http.BasicAuth{ + auth = &githttp.BasicAuth{ Username: user, Password: pass, } diff --git a/ops/sync-gcsr/manifest.yaml b/ops/sync-gcsr/manifest.yaml new file mode 100644 index 000000000000..3016c2ca57ee --- /dev/null +++ b/ops/sync-gcsr/manifest.yaml @@ -0,0 +1,23 @@ +image: nixos/latest +sources: + - https://git.tazj.in/ +secrets: + # cachix/tazjin + - f7f02546-4d95-44f7-a98e-d61fdded8b5b +tasks: + - setup: | + # sourcehut does not censor secrets in builds, hence this hack: + echo -n 'export CACHIX_SIGNING_KEY=' > cachix-preamble + cat cachix-preamble ~/.cachix-tazjin >> ~/.buildenv + nix-env -iA third_party.cachix -f git.tazj.in + cachix use tazjin + - build: | + cd git.tazj.in + nix-build ci-builds.nix > built-paths + - cache: | + cd git.tazj.in + cat built-paths | cachix push tazjin +triggers: + - action: email + condition: failure + to: mail@tazj.in |