diff options
Diffstat (limited to 'third_party/overlays')
24 files changed, 884 insertions, 99 deletions
diff --git a/third_party/overlays/dhall/OWNERS b/third_party/overlays/dhall/OWNERS index a742d0d22b..a640227914 100644 --- a/third_party/overlays/dhall/OWNERS +++ b/third_party/overlays/dhall/OWNERS @@ -1,3 +1 @@ -inherited: true -owners: - - Profpatsch +Profpatsch diff --git a/third_party/overlays/dhall/default.nix b/third_party/overlays/dhall/default.nix index 74d2a999b9..4625035999 100644 --- a/third_party/overlays/dhall/default.nix +++ b/third_party/overlays/dhall/default.nix @@ -1,26 +1,29 @@ { ... }: -pkgs: _: +self: super: let + # binary releases of dhall tools, since the build in nixpkgs is # broken most of the time. The binaries are also fully static # builds, instead of the half-static crap that nixpkgs produces. easy-dhall-nix = import (builtins.fetchTarball { - url = "https://github.com/justinwoo/easy-dhall-nix/archive/eae7f64c4d6c70681e5a56c84198236930ba425e.tar.gz"; - sha256 = "1y2x15v8a679vlpxazjpibfwajp6zph60f8wjcm4xflbvazk0dx7"; + url = "https://github.com/justinwoo/easy-dhall-nix/archive/dce9acbb99776a7f1344db4751d6080380f76f57.tar.gz"; + sha256 = "0ckp6515gfvbxm08yyll87d9vg8sq2l21gwav2npzvwc3xz2lccf"; }) - { inherit pkgs; }; + { pkgs = self; }; in { - dhall = easy-dhall-nix.dhall-simple; + # ATTN: see the haskell overlay for some overrides we need. + + # dhall = easy-dhall-nix.dhall-simple; + # dhall-nix = easy-dhall-nix.dhall-nix-simple; dhall-bash = easy-dhall-nix.dhall-bash-simple; dhall-docs = easy-dhall-nix.dhall-docs-simple; dhall-json = easy-dhall-nix.dhall-json-simple; dhall-lsp-server = easy-dhall-nix.dhall-lsp-simple; - dhall-nix = easy-dhall-nix.dhall-nix-simple; # not yet in dhall-simple # dhall-nixpkgs = easy-dhall-nix.dhall-nixpkgs-simple; dhall-yaml = easy-dhall-nix.dhall-yaml-simple; diff --git a/third_party/overlays/ecl-static.nix b/third_party/overlays/ecl-static.nix index 66579c33ab..d81075bdee 100644 --- a/third_party/overlays/ecl-static.nix +++ b/third_party/overlays/ecl-static.nix @@ -20,15 +20,6 @@ self: super: ecl-static = (super.pkgsMusl.ecl.override { inherit (self.pkgsStatic) gmp libffi boehmgc; }).overrideAttrs (drv: rec { - # version must not be changed as it indicates where to find the bundled libs, - # using ecl HEAD is necessary for us since it includes multiple fixes to do - # with bytecode compilation and allows to concatenate fasc files again. - src = self.fetchFromGitLab { - owner = "embeddable-common-lisp"; - repo = "ecl"; - rev = "1c989247c1b0bf1d38a76aec30b9ca5e41afe1e3"; - sha256 = "0bzjqw6m1kk5z5b81yizic347k931msp5lf78x65dcw3fqfwv3xn"; - }; configureFlags = drv.configureFlags ++ [ "--disable-shared" "--with-dffi=no" # will fail at runtime anyways if statically linked diff --git a/third_party/overlays/emacs.nix b/third_party/overlays/emacs.nix deleted file mode 100644 index 341feb5015..0000000000 --- a/third_party/overlays/emacs.nix +++ /dev/null @@ -1,4 +0,0 @@ -# Emacs overlay from https://github.com/nix-community/emacs-overlay -{ depot, ... }: - -import depot.third_party.sources.emacs-overlay diff --git a/third_party/overlays/haskell/OWNERS b/third_party/overlays/haskell/OWNERS new file mode 100644 index 0000000000..5f87d2f271 --- /dev/null +++ b/third_party/overlays/haskell/OWNERS @@ -0,0 +1,2 @@ +Profpatsch +sterni diff --git a/third_party/overlays/haskell/default.nix b/third_party/overlays/haskell/default.nix index 0ed0196a28..dc1201ec43 100644 --- a/third_party/overlays/haskell/default.nix +++ b/third_party/overlays/haskell/default.nix @@ -7,25 +7,69 @@ self: super: # overlay parameters for the nixpkgs overlay let - overrides = hsSelf: hsSuper: with self.haskell.lib.compose; { - # No overrides for the default package set necessary at the moment - }; + haskellLib = self.haskell.lib.compose; in { haskellPackages = super.haskellPackages.override { - inherit overrides; + overrides = hsSelf: hsSuper: { + punycode = haskellLib.appendPatch + (self.fetchpatch { + name = "punycode-mtl-2.3.patch"; + url = "https://github.com/litherum/punycode/pull/5/commits/41e55c8b7cef14563e6d04a7190dbabff5a77886.patch"; + sha256 = "03kgmy4z36jv16ffp5jrig2gr8ydc8cl1iscc7difisaq88mxvqc"; + }) + hsSuper.punycode; + + # Build with deprecated ansi-wl-pprint is broken now, use HEAD which switched to + # prettyprinter + tmp-postgres = haskellLib.overrideSrc + { + version = "unstable-2023-08-08"; + src = self.fetchFromGitHub { + owner = "jfischoff"; + repo = "tmp-postgres"; + rev = "7f2467a6d6d5f6db7eed59919a6773fe006cf22b"; + sha256 = "0l1gdx5s8ximgawd3yzfy47pv5pgwqmjqp8hx5rbrq68vr04wkbl"; + }; + } + (hsSuper.tmp-postgres.override { + ansi-wl-pprint = hsSelf.prettyprinter; + }); + + ihp-hsx = lib.pipe hsSuper.ihp-hsx [ + (haskellLib.overrideSrc { + version = "unstable-2023-03-28"; + src = "${self.fetchFromGitHub { + owner = "digitallyinduced"; + repo = "ihp"; + rev = "ab4ecd05f4e7b6b3c4b74b82d39fc6c5cc48766b"; + sha256 = "1fj5q9lygnmvqqv2fwqdj12sv63gkdfv5ha6fi190sv07dp9n9an"; + }}/ihp-hsx"; + }) + haskellLib.doJailbreak + ]; + + pa-prelude = hsSelf.callPackage ./extra-pkgs/pa-prelude.nix { }; + pa-error-tree = hsSelf.callPackage ./extra-pkgs/pa-error-tree-0.1.0.0.nix { }; + pa-field-parser = hsSelf.callPackage ./extra-pkgs/pa-field-parser.nix { }; + pa-label = hsSelf.callPackage ./extra-pkgs/pa-label.nix { }; + pa-pretty = hsSelf.callPackage ./extra-pkgs/pa-pretty-0.1.1.0.nix { }; + pa-json = hsSelf.callPackage ./extra-pkgs/pa-json.nix { }; + pa-run-command = hsSelf.callPackage ./extra-pkgs/pa-run-command-0.1.0.0.nix { }; + }; }; haskell = lib.recursiveUpdate super.haskell { packages.ghc8107 = super.haskell.packages.ghc8107.override { - overrides = lib.composeExtensions overrides ( - hsSelf: hsSuper: with self.haskell.lib.compose; { - # TODO(sterni): TODO(grfn): patch xanthous to work with random-fu 0.3.*, - # so we can use GHC 9.0.2 and benefit from upstream binary cache. - random-fu = hsSelf.callPackage ./extra-pkgs/random-fu-0.2.nix { }; - rvar = hsSelf.callPackage ./extra-pkgs/rvar-0.2.nix { }; - } - ); + overrides = hsSelf: hsSuper: { + # TODO(sterni): TODO(grfn): patch xanthous to work with random-fu 0.3.*, + # so we can use GHC 9.0.2 and benefit from upstream binary cache. + random-fu = hsSelf.callPackage ./extra-pkgs/random-fu-0.2.nix { }; + rvar = hsSelf.callPackage ./extra-pkgs/rvar-0.2.nix { }; + + # TODO(grfn): port to brick 1.4 (EventM gains an additional type argument in 1.0) + brick = hsSelf.callPackage ./extra-pkgs/brick-0.73.nix { }; + }; }; }; } diff --git a/third_party/overlays/haskell/extra-pkgs/brick-0.73.nix b/third_party/overlays/haskell/extra-pkgs/brick-0.73.nix new file mode 100644 index 0000000000..c5e2883c75 --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/brick-0.73.nix @@ -0,0 +1,70 @@ +{ mkDerivation +, base +, bytestring +, config-ini +, containers +, contravariant +, data-clist +, deepseq +, directory +, dlist +, exceptions +, filepath +, lib +, microlens +, microlens-mtl +, microlens-th +, QuickCheck +, stm +, template-haskell +, text +, text-zipper +, transformers +, unix +, vector +, vty +, word-wrap +}: +mkDerivation { + pname = "brick"; + version = "0.73"; + sha256 = "741c8d0717f0ab5addd5d3acc88cb36d645a0c73907bde509b2fd9d9bc02039c"; + isLibrary = true; + isExecutable = true; + libraryHaskellDepends = [ + base + bytestring + config-ini + containers + contravariant + data-clist + deepseq + directory + dlist + exceptions + filepath + microlens + microlens-mtl + microlens-th + stm + template-haskell + text + text-zipper + transformers + unix + vector + vty + word-wrap + ]; + testHaskellDepends = [ + base + containers + microlens + QuickCheck + vector + vty + ]; + homepage = "https://github.com/jtdaugherty/brick/"; + description = "A declarative terminal user interface library"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/pa-error-tree-0.1.0.0.nix b/third_party/overlays/haskell/extra-pkgs/pa-error-tree-0.1.0.0.nix new file mode 100644 index 0000000000..a38cd4efaa --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/pa-error-tree-0.1.0.0.nix @@ -0,0 +1,10 @@ +{ mkDerivation, base, containers, lib, pa-prelude }: +mkDerivation { + pname = "pa-error-tree"; + version = "0.1.0.0"; + sha256 = "f82d3d905e8d9f0d31c81f31c424b9a95c65a8925517ccac92134f410cf8d639"; + libraryHaskellDepends = [ base containers pa-prelude ]; + homepage = "https://github.com/possehl-analytics/pa-hackage"; + description = "Collect a tree of errors and pretty-print"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/pa-field-parser.nix b/third_party/overlays/haskell/extra-pkgs/pa-field-parser.nix new file mode 100644 index 0000000000..a3c146ee09 --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/pa-field-parser.nix @@ -0,0 +1,39 @@ +{ mkDerivation +, aeson +, aeson-better-errors +, attoparsec +, base +, case-insensitive +, containers +, lib +, pa-error-tree +, pa-prelude +, scientific +, semigroupoids +, template-haskell +, text +, time +}: +mkDerivation { + pname = "pa-field-parser"; + version = "0.3.0.0"; + sha256 = "528c2b6bf5ad6454861b059c7eb6924f4c32bcb5b8faa4c2389d9ddfd92fcd57"; + libraryHaskellDepends = [ + aeson + aeson-better-errors + attoparsec + base + case-insensitive + containers + pa-error-tree + pa-prelude + scientific + semigroupoids + template-haskell + text + time + ]; + homepage = "https://github.com/possehl-analytics/pa-hackage"; + description = "“Vertical” parsing of values"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/pa-json.nix b/third_party/overlays/haskell/extra-pkgs/pa-json.nix new file mode 100644 index 0000000000..8ce838b22c --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/pa-json.nix @@ -0,0 +1,43 @@ +{ mkDerivation +, aeson +, aeson-better-errors +, aeson-pretty +, base +, base64-bytestring +, bytestring +, containers +, lib +, pa-error-tree +, pa-field-parser +, pa-label +, pa-prelude +, scientific +, text +, time +, vector +}: +mkDerivation { + pname = "pa-json"; + version = "0.3.0.0"; + sha256 = "45e79765e57e21400f3f3b1e86094473fac61d298618d7e34f6cad4988d8923b"; + libraryHaskellDepends = [ + aeson + aeson-better-errors + aeson-pretty + base + base64-bytestring + bytestring + containers + pa-error-tree + pa-field-parser + pa-label + pa-prelude + scientific + text + time + vector + ]; + homepage = "https://github.com/possehl-analytics/pa-hackage"; + description = "Our JSON parsers/encoders"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/pa-label.nix b/third_party/overlays/haskell/extra-pkgs/pa-label.nix new file mode 100644 index 0000000000..7cfa257c81 --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/pa-label.nix @@ -0,0 +1,10 @@ +{ mkDerivation, base, lib }: +mkDerivation { + pname = "pa-label"; + version = "0.1.1.0"; + sha256 = "b40183900c045641c0632ed8e53a326c0c0e9c2806568613c03b3131d9016183"; + libraryHaskellDepends = [ base ]; + homepage = "https://github.com/possehl-analytics/pa-hackage"; + description = "Labels, and labelled tuples and enums (GHC >9.2)"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/pa-prelude.nix b/third_party/overlays/haskell/extra-pkgs/pa-prelude.nix new file mode 100644 index 0000000000..17e1996ab6 --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/pa-prelude.nix @@ -0,0 +1,43 @@ +{ mkDerivation +, base +, bytestring +, containers +, error +, exceptions +, lib +, mtl +, profunctors +, PyF +, scientific +, semigroupoids +, template-haskell +, text +, these +, validation-selective +, vector +}: +mkDerivation { + pname = "pa-prelude"; + version = "0.2.0.0"; + sha256 = "68015f7c19e9c618fc04e2516baccfce52af24efb9ca1480162c9ea0aef7f301"; + libraryHaskellDepends = [ + base + bytestring + containers + error + exceptions + mtl + profunctors + PyF + scientific + semigroupoids + template-haskell + text + these + validation-selective + vector + ]; + homepage = "https://github.com/possehl-analytics/pa-hackage"; + description = "The Possehl Analytics Prelude"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/pa-pretty-0.1.1.0.nix b/third_party/overlays/haskell/extra-pkgs/pa-pretty-0.1.1.0.nix new file mode 100644 index 0000000000..d6dadef849 --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/pa-pretty-0.1.1.0.nix @@ -0,0 +1,29 @@ +{ mkDerivation +, aeson +, aeson-pretty +, ansi-terminal +, base +, hscolour +, lib +, nicify-lib +, pa-prelude +, text +}: +mkDerivation { + pname = "pa-pretty"; + version = "0.1.1.0"; + sha256 = "da925a7cf2ac49c5769d7ebd08c2599b537efe45b3d506bf4d7c8673633ef6c9"; + libraryHaskellDepends = [ + aeson + aeson-pretty + ansi-terminal + base + hscolour + nicify-lib + pa-prelude + text + ]; + homepage = "https://github.com/possehl-analytics/pa-hackage"; + description = "Some pretty-printing helpers"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/pa-run-command-0.1.0.0.nix b/third_party/overlays/haskell/extra-pkgs/pa-run-command-0.1.0.0.nix new file mode 100644 index 0000000000..b12eb5efbf --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/pa-run-command-0.1.0.0.nix @@ -0,0 +1,25 @@ +{ mkDerivation +, base +, bytestring +, lib +, monad-logger +, pa-prelude +, text +, typed-process +}: +mkDerivation { + pname = "pa-run-command"; + version = "0.1.0.0"; + sha256 = "37837e0cddedc9b615063f0357115739c53b5dcb8af82ce86a95a3a5c88c29a3"; + libraryHaskellDepends = [ + base + bytestring + monad-logger + pa-prelude + text + typed-process + ]; + homepage = "https://github.com/possehl-analytics/pa-hackage"; + description = "Helper functions for spawning subprocesses"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/patches/.skip-tree b/third_party/overlays/patches/.skip-tree new file mode 100644 index 0000000000..86eae51a6d --- /dev/null +++ b/third_party/overlays/patches/.skip-tree @@ -0,0 +1 @@ +No readTree-compatible files. diff --git a/third_party/overlays/patches/0001-configure-ac-version.patch b/third_party/overlays/patches/0001-configure-ac-version.patch new file mode 100644 index 0000000000..fa2575cb93 --- /dev/null +++ b/third_party/overlays/patches/0001-configure-ac-version.patch @@ -0,0 +1,13 @@ +diff --git a/configure.ac b/configure.ac +index e861e42..018c19c 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -26,7 +26,7 @@ + #;**********************************************************************; + + AC_INIT([tpm2-pkcs11], +- [m4_esyscmd_s([git describe --tags --always --dirty])], ++ [git-@VERSION@], + [https://github.com/tpm2-software/tpm2-pkcs11/issues], + [], + [https://github.com/tpm2-software/tpm2-pkcs11]) diff --git a/third_party/overlays/patches/buf-tests-dont-use-file-transport.patch b/third_party/overlays/patches/buf-tests-dont-use-file-transport.patch new file mode 100644 index 0000000000..34be80eb36 --- /dev/null +++ b/third_party/overlays/patches/buf-tests-dont-use-file-transport.patch @@ -0,0 +1,64 @@ +commit e9219b88de5ed37af337ee2d2e71e7ec7c0aad1b +Author: Robbert van Ginkel <rvanginkel@buf.build> +Date: Thu Oct 20 16:43:28 2022 -0400 + + Fix git unit test by using fake git server rather than file:// (#1518) + + More recent versions of git fix a CVE by disabling some usage of the + `file://` transport, see + https://github.blog/2022-10-18-git-security-vulnerabilities-announced/#cve-2022-39253. + We were using this transport in tests. + + Instead, use https://git-scm.com/docs/git-http-backend to serve up this + repository locally so we don't have to use the file protocol. This + should be a more accurate tests, since we mostly expect submodules to + come from servers. + +diff --git a/.golangci.yml b/.golangci.yml +index 318d1171..865e03e7 100644 +--- a/.golangci.yml ++++ b/.golangci.yml +@@ -136,3 +136,8 @@ issues: + - linters: + - containedctx + path: private/bufpkg/bufmodule/bufmoduleprotocompile ++ # We should be able to use net/http/cgi in a unit test, in addition the CVE mentions only versions of go < 1.6.3 are affected. ++ - linters: ++ - gosec ++ path: private/pkg/git/git_test.go ++ text: "G504:" +diff --git a/private/pkg/git/git_test.go b/private/pkg/git/git_test.go +index 7b77b6cd..7132054e 100644 +--- a/private/pkg/git/git_test.go ++++ b/private/pkg/git/git_test.go +@@ -17,6 +17,8 @@ package git + import ( + "context" + "errors" ++ "net/http/cgi" ++ "net/http/httptest" + "os" + "os/exec" + "path/filepath" +@@ -213,6 +215,21 @@ func createGitDirs( + runCommand(ctx, t, container, runner, "git", "-C", submodulePath, "add", "test.proto") + runCommand(ctx, t, container, runner, "git", "-C", submodulePath, "commit", "-m", "commit 0") + ++ gitExecPath, err := command.RunStdout(ctx, container, runner, "git", "--exec-path") ++ require.NoError(t, err) ++ t.Log(filepath.Join(string(gitExecPath), "git-http-backend")) ++ // https://git-scm.com/docs/git-http-backend#_description ++ f, err := os.Create(filepath.Join(submodulePath, ".git", "git-daemon-export-ok")) ++ require.NoError(t, err) ++ require.NoError(t, f.Close()) ++ server := httptest.NewServer(&cgi.Handler{ ++ Path: filepath.Join(strings.TrimSpace(string(gitExecPath)), "git-http-backend"), ++ Dir: submodulePath, ++ Env: []string{"GIT_PROJECT_ROOT=" + submodulePath}, ++ }) ++ t.Cleanup(server.Close) ++ submodulePath = server.URL ++ + originPath := filepath.Join(tmpDir, "origin") + require.NoError(t, os.MkdirAll(originPath, 0777)) + runCommand(ctx, t, container, runner, "git", "-C", originPath, "init") diff --git a/third_party/overlays/patches/cbtemulator-uds.patch b/third_party/overlays/patches/cbtemulator-uds.patch new file mode 100644 index 0000000000..a19255306f --- /dev/null +++ b/third_party/overlays/patches/cbtemulator-uds.patch @@ -0,0 +1,140 @@ +commit 1397e10225d8c6fd079a86fccd58fb5d0f4200bc +Author: Florian Klink <flokli@flokli.de> +Date: Fri Mar 29 10:06:34 2024 +0100 + + feat(bigtable/emulator): allow listening on Unix Domain Sockets + + cbtemulator listening on unix domain sockets is much easier than trying + to allocate free TCP ports, especially if many cbtemulators are run at + the same time in integration tests. + + This adds an additional flag, address, which has priority if it's set, + rather than host:port. + + `NewServer` already takes a `laddr string`, so we simply check for it to + contain slashes, and if so, listen on unix, rather than TCP. + +diff --git a/bigtable/bttest/inmem.go b/bigtable/bttest/inmem.go +index 556abc2a85..33e4bf2667 100644 +--- a/bttest/inmem.go ++++ b/bttest/inmem.go +@@ -40,6 +40,7 @@ import ( + "math" + "math/rand" + "net" ++ "os" + "regexp" + "sort" + "strings" +@@ -106,7 +107,15 @@ type server struct { + // The Server will be listening for gRPC connections, without TLS, + // on the provided address. The resolved address is named by the Addr field. + func NewServer(laddr string, opt ...grpc.ServerOption) (*Server, error) { +- l, err := net.Listen("tcp", laddr) ++ var l net.Listener ++ var err error ++ ++ // If the address contains slashes, listen on a unix domain socket instead. ++ if strings.Contains(laddr, "/") { ++ l, err = net.Listen("unix", laddr) ++ } else { ++ l, err = net.Listen("tcp", laddr) ++ } + if err != nil { + return nil, err + } +diff --git a/bigtable/cmd/emulator/cbtemulator.go b/bigtable/cmd/emulator/cbtemulator.go +index 144c09ffb1..deaf69b717 100644 +--- a/cmd/emulator/cbtemulator.go ++++ b/cmd/emulator/cbtemulator.go +@@ -27,8 +27,9 @@ import ( + ) + + var ( +- host = flag.String("host", "localhost", "the address to bind to on the local machine") +- port = flag.Int("port", 9000, "the port number to bind to on the local machine") ++ host = flag.String("host", "localhost", "the address to bind to on the local machine") ++ port = flag.Int("port", 9000, "the port number to bind to on the local machine") ++ address = flag.String("address", "", "address:port number or unix socket path to listen on. Has priority over host/port") + ) + + const ( +@@ -42,7 +43,15 @@ func main() { + grpc.MaxRecvMsgSize(maxMsgSize), + grpc.MaxSendMsgSize(maxMsgSize), + } +- srv, err := bttest.NewServer(fmt.Sprintf("%s:%d", *host, *port), opts...) ++ ++ var laddr string ++ if *address != "" { ++ laddr = *address ++ } else { ++ laddr = fmt.Sprintf("%s:%d", *host, *port) ++ } ++ ++ srv, err := bttest.NewServer(laddr, opts...) + if err != nil { + log.Fatalf("failed to start emulator: %v", err) + } +commit ce16f843d6c93159d86b3807c6d9ff66e43aac67 +Author: Florian Klink <flokli@flokli.de> +Date: Fri Mar 29 11:53:15 2024 +0100 + + feat(bigtable): clean up unix socket on close + + Call srv.Close when receiving an interrupt, and delete the unix domain + socket in that function. + +diff --git a/bigtable/bttest/inmem.go b/bigtable/bttest/inmem.go +index 33e4bf2667..0dc96024b1 100644 +--- a/bttest/inmem.go ++++ b/bttest/inmem.go +@@ -148,6 +148,11 @@ func (s *Server) Close() { + + s.srv.Stop() + s.l.Close() ++ ++ // clean up unix socket ++ if strings.Contains(s.Addr, "/") { ++ _ = os.Remove(s.Addr) ++ } + } + + func (s *server) CreateTable(ctx context.Context, req *btapb.CreateTableRequest) (*btapb.Table, error) { +diff --git a/bigtable/cmd/emulator/cbtemulator.go b/bigtable/cmd/emulator/cbtemulator.go +index deaf69b717..5a9e8f7a8c 100644 +--- a/cmd/emulator/cbtemulator.go ++++ b/cmd/emulator/cbtemulator.go +@@ -18,9 +18,12 @@ cbtemulator launches the in-memory Cloud Bigtable server on the given address. + package main + + import ( ++ "context" + "flag" + "fmt" + "log" ++ "os" ++ "os/signal" + + "cloud.google.com/go/bigtable/bttest" + "google.golang.org/grpc" +@@ -51,11 +54,18 @@ func main() { + laddr = fmt.Sprintf("%s:%d", *host, *port) + } + ++ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) ++ defer stop() ++ + srv, err := bttest.NewServer(laddr, opts...) + if err != nil { + log.Fatalf("failed to start emulator: %v", err) + } + + fmt.Printf("Cloud Bigtable emulator running on %s\n", srv.Addr) +- select {} ++ select { ++ case <-ctx.Done(): ++ srv.Close() ++ stop() ++ } + } diff --git a/third_party/overlays/patches/clickhouse-support-reading-arrow-LargeListArray.patch b/third_party/overlays/patches/clickhouse-support-reading-arrow-LargeListArray.patch new file mode 100644 index 0000000000..9e79aa7267 --- /dev/null +++ b/third_party/overlays/patches/clickhouse-support-reading-arrow-LargeListArray.patch @@ -0,0 +1,106 @@ +From cdea2e8ad98995202ce81c9c030f2ae64d73b05a Mon Sep 17 00:00:00 2001 +From: edef <edef@edef.eu> +Date: Mon, 30 Oct 2023 08:08:10 +0000 +Subject: [PATCH] Support reading arrow::LargeListArray + +--- + .../Formats/Impl/ArrowColumnToCHColumn.cpp | 33 +++++++++++++++---- + 1 file changed, 26 insertions(+), 7 deletions(-) + +diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +index 6f9d49498f2..b93846cd4eb 100644 +--- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp ++++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +@@ -436,6 +436,22 @@ static ColumnPtr readByteMapFromArrowColumn(std::shared_ptr<arrow::ChunkedArray> + return nullmap_column; + } + ++template <typename T> ++struct ArrowOffsetArray; ++ ++template <> ++struct ArrowOffsetArray<arrow::ListArray> ++{ ++ using type = arrow::Int32Array; ++}; ++ ++template <> ++struct ArrowOffsetArray<arrow::LargeListArray> ++{ ++ using type = arrow::Int64Array; ++}; ++ ++template <typename ArrowListArray> + static ColumnPtr readOffsetsFromArrowListColumn(std::shared_ptr<arrow::ChunkedArray> & arrow_column) + { + auto offsets_column = ColumnUInt64::create(); +@@ -444,9 +460,9 @@ static ColumnPtr readOffsetsFromArrowListColumn(std::shared_ptr<arrow::ChunkedAr + + for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i) + { +- arrow::ListArray & list_chunk = dynamic_cast<arrow::ListArray &>(*(arrow_column->chunk(chunk_i))); ++ ArrowListArray & list_chunk = dynamic_cast<ArrowListArray &>(*(arrow_column->chunk(chunk_i))); + auto arrow_offsets_array = list_chunk.offsets(); +- auto & arrow_offsets = dynamic_cast<arrow::Int32Array &>(*arrow_offsets_array); ++ auto & arrow_offsets = dynamic_cast<ArrowOffsetArray<ArrowListArray>::type &>(*arrow_offsets_array); + + /* + * CH uses element size as "offsets", while arrow uses actual offsets as offsets. +@@ -602,13 +618,14 @@ static ColumnPtr readColumnWithIndexesData(std::shared_ptr<arrow::ChunkedArray> + } + } + ++template <typename ArrowListArray> + static std::shared_ptr<arrow::ChunkedArray> getNestedArrowColumn(std::shared_ptr<arrow::ChunkedArray> & arrow_column) + { + arrow::ArrayVector array_vector; + array_vector.reserve(arrow_column->num_chunks()); + for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i) + { +- arrow::ListArray & list_chunk = dynamic_cast<arrow::ListArray &>(*(arrow_column->chunk(chunk_i))); ++ ArrowListArray & list_chunk = dynamic_cast<ArrowListArray &>(*(arrow_column->chunk(chunk_i))); + + /* + * It seems like arrow::ListArray::values() (nested column data) might or might not be shared across chunks. +@@ -819,12 +836,12 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( + key_type_hint = map_type_hint->getKeyType(); + } + } +- auto arrow_nested_column = getNestedArrowColumn(arrow_column); ++ auto arrow_nested_column = getNestedArrowColumn<arrow::ListArray>(arrow_column); + auto nested_column = readColumnFromArrowColumn(arrow_nested_column, column_name, format_name, false, dictionary_infos, allow_null_type, skip_columns_with_unsupported_types, skipped, date_time_overflow_behavior, nested_type_hint, true); + if (skipped) + return {}; + +- auto offsets_column = readOffsetsFromArrowListColumn(arrow_column); ++ auto offsets_column = readOffsetsFromArrowListColumn<arrow::ListArray>(arrow_column); + + const auto * tuple_column = assert_cast<const ColumnTuple *>(nested_column.column.get()); + const auto * tuple_type = assert_cast<const DataTypeTuple *>(nested_column.type.get()); +@@ -846,7 +863,9 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( + return {std::move(map_column), std::move(map_type), column_name}; + } + case arrow::Type::LIST: ++ case arrow::Type::LARGE_LIST: + { ++ bool is_large = arrow_column->type()->id() == arrow::Type::LARGE_LIST; + DataTypePtr nested_type_hint; + if (type_hint) + { +@@ -854,11 +873,11 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( + if (array_type_hint) + nested_type_hint = array_type_hint->getNestedType(); + } +- auto arrow_nested_column = getNestedArrowColumn(arrow_column); ++ auto arrow_nested_column = is_large ? getNestedArrowColumn<arrow::LargeListArray>(arrow_column) : getNestedArrowColumn<arrow::ListArray>(arrow_column); + auto nested_column = readColumnFromArrowColumn(arrow_nested_column, column_name, format_name, false, dictionary_infos, allow_null_type, skip_columns_with_unsupported_types, skipped, date_time_overflow_behavior, nested_type_hint); + if (skipped) + return {}; +- auto offsets_column = readOffsetsFromArrowListColumn(arrow_column); ++ auto offsets_column = is_large ? readOffsetsFromArrowListColumn<arrow::LargeListArray>(arrow_column) : readOffsetsFromArrowListColumn<arrow::ListArray>(arrow_column); + auto array_column = ColumnArray::create(nested_column.column, offsets_column); + auto array_type = std::make_shared<DataTypeArray>(nested_column.type); + return {std::move(array_column), std::move(array_type), column_name}; +-- +2.42.0 + diff --git a/third_party/overlays/patches/crate2nix-run-tests-in-build-source.patch b/third_party/overlays/patches/crate2nix-run-tests-in-build-source.patch new file mode 100644 index 0000000000..52793270e6 --- /dev/null +++ b/third_party/overlays/patches/crate2nix-run-tests-in-build-source.patch @@ -0,0 +1,69 @@ +From 7cf084f73f7d15fe0538a625182fa7179c083b3d Mon Sep 17 00:00:00 2001 +From: Raito Bezarius <masterancpp@gmail.com> +Date: Tue, 16 Jan 2024 02:10:48 +0100 +Subject: [PATCH] fix(template): run tests in `/build/source` instead `/build` + +Previously, the source tree was located inline in `/build` during tests, this was a mistake +because the crates more than often are built in `/build/source` as per the `sourceRoot` system. + +This can cause issues with test binaries hardcoding `/build/source/...` as their choice for doing things, +causing them to be confused in the test phase which is relocated without rewriting the paths inside test binaries. + +We fix that by relocating ourselves in the right hierarchy. + +This is a "simple" fix in the sense that more edge cases could exist but they are hard to reason about +because they would be crates using custom `sourceRoot`, i.e. having `crate.sourceRoot` set and then it becomes +a bit hard to reproduce the hierarchy, you need to analyze whether the path is absolute or relative, + +If it's relative, you can just reuse it and reproduce that specific hierarchy. +If it's absolute, you need to cut the "absolute" meaningless part, e.g. `$NIX_BUILD_TOP/` and proceed like +it's a relative path IMHO. +--- + crate2nix/Cargo.nix | 10 ++++++++++ + crate2nix/templates/nix/crate2nix/default.nix | 10 ++++++++++ + +diff --git a/Cargo.nix b/Cargo.nix +index 6ef7a49..172ff34 100644 +--- a/Cargo.nix ++++ b/Cargo.nix +@@ -2889,6 +2889,16 @@ rec { + # recreate a file hierarchy as when running tests with cargo + + # the source for test data ++ # It's necessary to locate the source in $NIX_BUILD_TOP/source/ ++ # instead of $NIX_BUILD_TOP/ ++ # because we compiled those test binaries in the former and not the latter. ++ # So all paths will expect source tree to be there and not in the build top directly. ++ # For example: $NIX_BUILD_TOP := /build in general, if you ask yourself. ++ # TODO(raitobezarius): I believe there could be more edge cases if `crate.sourceRoot` ++ # do exist but it's very hard to reason about them, so let's wait until the first bug report. ++ mkdir -p source/ ++ cd source/ ++ + ${pkgs.buildPackages.xorg.lndir}/bin/lndir ${crate.src} + + # build outputs +diff --git a/crate2nix/templates/nix/crate2nix/default.nix b/crate2nix/templates/nix/crate2nix/default.nix +index e4fc2e9..dfb14c4 100644 +--- a/templates/nix/crate2nix/default.nix ++++ b/templates/nix/crate2nix/default.nix +@@ -135,6 +135,16 @@ rec { + # recreate a file hierarchy as when running tests with cargo + + # the source for test data ++ # It's necessary to locate the source in $NIX_BUILD_TOP/source/ ++ # instead of $NIX_BUILD_TOP/ ++ # because we compiled those test binaries in the former and not the latter. ++ # So all paths will expect source tree to be there and not in the build top directly. ++ # For example: $NIX_BUILD_TOP := /build in general, if you ask yourself. ++ # TODO(raitobezarius): I believe there could be more edge cases if `crate.sourceRoot` ++ # do exist but it's very hard to reason about them, so let's wait until the first bug report. ++ mkdir -p source/ ++ cd source/ ++ + ${pkgs.buildPackages.xorg.lndir}/bin/lndir ${crate.src} + + # build outputs +-- +2.43.0 + diff --git a/third_party/overlays/patches/crate2nix-tests-debug.patch b/third_party/overlays/patches/crate2nix-tests-debug.patch new file mode 100644 index 0000000000..384178c805 --- /dev/null +++ b/third_party/overlays/patches/crate2nix-tests-debug.patch @@ -0,0 +1,12 @@ +diff --git a/templates/nix/crate2nix/default.nix b/templates/nix/crate2nix/default.nix +index 4eefda8..d064118 100644 +--- a/templates/nix/crate2nix/default.nix ++++ b/templates/nix/crate2nix/default.nix +@@ -111,6 +111,7 @@ rec { + ( + _: { + buildTests = true; ++ release = false; + } + ); + # If the user hasn't set any pre/post commands, we don't want to diff --git a/third_party/overlays/patches/evans-add-support-for-unix-domain-sockets.patch b/third_party/overlays/patches/evans-add-support-for-unix-domain-sockets.patch new file mode 100644 index 0000000000..c66528f538 --- /dev/null +++ b/third_party/overlays/patches/evans-add-support-for-unix-domain-sockets.patch @@ -0,0 +1,39 @@ +From 55d7e7af7c56f678eb817059417241bb61ee5181 Mon Sep 17 00:00:00 2001 +From: Florian Klink <flokli@flokli.de> +Date: Sun, 8 Oct 2023 11:00:27 +0200 +Subject: [PATCH] add support for unix domain sockets + +grpc.NewClient already supports connecting to unix domain sockets, and +accepts a string anyways. + +As a quick fix, detect the `address` starting with `unix://` and don't +add the port. + +In the long term, we might want to deprecate `host` and `port` cmdline +args in favor of a single `address` arg. +--- + mode/common.go | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/mode/common.go b/mode/common.go +index dfc7839..55f1e36 100644 +--- a/mode/common.go ++++ b/mode/common.go +@@ -13,7 +13,13 @@ import ( + ) + + func newGRPCClient(cfg *config.Config) (grpc.Client, error) { +- addr := fmt.Sprintf("%s:%s", cfg.Server.Host, cfg.Server.Port) ++ addr := cfg.Server.Host ++ ++ // as long as the address doesn't start with unix, also add the port. ++ if !strings.HasPrefix(cfg.Server.Host, "unix://") { ++ addr = fmt.Sprintf("%s:%s", cfg.Server.Host, cfg.Server.Port) ++ } ++ + if cfg.Request.Web { + //TODO: remove second arg + return grpc.NewWebClient(addr, cfg.Server.Reflection, false, "", "", "", grpc.Headers(cfg.Request.Header)), nil +-- +2.42.0 + diff --git a/third_party/overlays/patches/tpm2-pkcs11-190-dbupgrade.patch b/third_party/overlays/patches/tpm2-pkcs11-190-dbupgrade.patch new file mode 100644 index 0000000000..f831c11a80 --- /dev/null +++ b/third_party/overlays/patches/tpm2-pkcs11-190-dbupgrade.patch @@ -0,0 +1,29 @@ +From 987323794148a6ff5ce3d02eef8cfeb46bee1761 Mon Sep 17 00:00:00 2001 +From: Anton <tracefinder@gmail.com> +Date: Tue, 7 Nov 2023 12:02:15 +0300 +Subject: [PATCH] Skip null attribute during DB update + +Signed-off-by: Anton <tracefinder@gmail.com> +--- + src/lib/db.c | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +diff --git a/src/lib/db.c b/src/lib/db.c +index b4bbd1bf..74c5a7b4 100644 +--- a/src/lib/db.c ++++ b/src/lib/db.c +@@ -2169,9 +2169,11 @@ static CK_RV dbup_handler_from_7_to_8(sqlite3 *updb) { + + /* for each tobject */ + CK_ATTRIBUTE_PTR a = attr_get_attribute_by_type(tobj->attrs, CKA_ALLOWED_MECHANISMS); +- CK_BYTE type = type_from_ptr(a->pValue, a->ulValueLen); +- if (type != TYPE_BYTE_INT_SEQ) { +- rv = _db_update_tobject_attrs(updb, tobj->id, tobj->attrs); ++ if (a) { ++ CK_BYTE type = type_from_ptr(a->pValue, a->ulValueLen); ++ if (type != TYPE_BYTE_INT_SEQ) { ++ rv = _db_update_tobject_attrs(updb, tobj->id, tobj->attrs); ++ } + } + + tobject_free(tobj); diff --git a/third_party/overlays/tvl.nix b/third_party/overlays/tvl.nix index fbc48e1565..b54e899b88 100644 --- a/third_party/overlays/tvl.nix +++ b/third_party/overlays/tvl.nix @@ -1,41 +1,46 @@ # This overlay is used to make TVL-specific modifications in the # nixpkgs tree, where required. -{ depot, ... }: +{ lib, depot, localSystem, ... }: self: super: -let - # Rollback Nix to a stable version (2.3) with backports for - # build-user problems applied. - nixSrc = self.fetchFromGitHub - { - owner = "tvlfyi"; - repo = "nix"; - # branch 2.3-backport-await-users - rev = "880a62b08443a6baa55dab027b69bb8b1551a588"; - hash = "sha256:0jnwrzxh04d0pyhx4n8fg4w1w6ak48byl5k2i8j7fk4h9vd9649k"; - } // { revCount = 0; shortRev = "880a62b0"; }; -in -{ - nix = (import "${nixSrc}/release.nix" { - nix = nixSrc; - nixpkgs = super.path; - systems = [ builtins.currentSystem ]; - }).build."${builtins.currentSystem}"; - - clang-tools_11 = self.clang-tools.override { - llvmPackages = self.llvmPackages_11; +depot.nix.readTree.drvTargets { + nix_2_3 = (super.nix_2_3.override { + # flaky tests, long painful build, see https://github.com/NixOS/nixpkgs/pull/266443 + withAWS = false; + }); + nix = self.nix_2_3 // { + # avoid duplicate pipeline step + meta = self.nix_2_3.meta or { } // { + ci = self.nix_2_3.meta.ci or { } // { + skip = true; + }; + }; }; + nix_latest = super.nix.override ({ + # flaky tests, long painful build, see https://github.com/NixOS/nixpkgs/pull/266443 + withAWS = false; + }); + + # To match telega in emacs-overlay or wherever + tdlib = super.tdlib.overrideAttrs (_: { + version = "1.8.24"; + src = self.fetchFromGitHub { + owner = "tdlib"; + repo = "td"; + rev = "d79bd4b69403868897496da39b773ab25c69f6af"; + sha256 = "0bc5akzw12qwj45rzqkrhw65qlrn9q8pzmvc5aiqv4bvhkb1ghl0"; + }; + }); - # stdenv which uses clang, lld and libc++; full is a slight exaggeration, - # we for example don't use LLVM's libunwind - fullLlvm11Stdenv = self.overrideCC self.stdenv - (self.llvmPackages_11.libcxxStdenv.cc.override { - inherit (self.llvmPackages_11) bintools; - }); + home-manager = super.home-manager.overrideAttrs (_: { + src = depot.third_party.sources.home-manager; + version = "git-" + + builtins.substring 0 7 depot.third_party.sources.home-manager.rev; + }); # Add our Emacs packages to the fixpoint emacsPackagesFor = emacs: ( - (super.emacsPackagesFor emacs).overrideScope' (eself: esuper: { + (super.emacsPackagesFor emacs).overrideScope (eself: esuper: { tvlPackages = depot.tools.emacs-pkgs // depot.third_party.emacs; # Use the notmuch from nixpkgs instead of from the Emacs @@ -43,9 +48,7 @@ in notmuch = super.notmuch.emacs; # Build EXWM with the depot sources instead. - exwm = esuper.exwm.overrideAttrs (_: { - src = depot.path.origSrc + "/third_party/exwm"; - }); + depotExwm = eself.callPackage depot.third_party.exwm.override { }; # Workaround for magit checking the git version at load time magit = esuper.magit.overrideAttrs (_: { @@ -53,6 +56,19 @@ in self.git ]; }); + + # Pin xelb to a newer one until the new maintainers do a release. + xelb = eself.trivialBuild { + pname = "xelb"; + version = "0.19-dev"; # invented version, last actual release was 0.18 + + src = self.fetchFromGitHub { + owner = "emacs-exwm"; + repo = "xelb"; + rev = "86089eba2de6c818bfa2fac075cb7ad876262798"; + sha256 = "1mmlrd2zpcwiv8gh10y7lrpflnbmsycdascrxjr3bfcwa8yx7901"; + }; + }; }) ); @@ -65,7 +81,7 @@ in # nix-serve does not work with nix 2.4 # https://github.com/edolstra/nix-serve/issues/28 - nix-serve = super.nix-serve.override { nix = super.nix_2_3; }; + nix-serve = super.nix-serve.override { nix = self.nix_2_3; }; # Avoid builds of mkShell derivations in CI. mkShell = super.lib.makeOverridable (args: (super.mkShell args).overrideAttrs (_: { @@ -74,41 +90,34 @@ in }; })); - # bump nixpkgs-fmt to a version that doesn't touch whitespace in - # strings - nixpkgs-fmt = super.nixpkgs-fmt.overrideAttrs (old: rec { - src = self.fetchFromGitHub { - owner = "nix-community"; - repo = "nixpkgs-fmt"; - rev = "5ae8532b82eb040ca6b21ae2d02d9e88f604e76a"; - sha256 = "0hjkbcgz62793hzfzlaxyah8a2c1k79n1k891lg7kfw8mkbq0w4p"; - }; + # https://github.com/googleapis/google-cloud-go/pull/9665 + cbtemulator = super.cbtemulator.overrideAttrs (old: { + patches = old.patches or [ ] ++ [ + ./patches/cbtemulator-uds.patch + ]; + }); - cargoDeps = old.cargoDeps.overrideAttrs (_: { - inherit src; - outputHash = "10if2lmv8d95j3walq3ggx3y423yfy4yl9vplw3apd0s671bly8b"; - }); + crate2nix = super.crate2nix.overrideAttrs (old: { + patches = old.patches or [ ] ++ [ + # https://github.com/nix-community/crate2nix/pull/301 + ./patches/crate2nix-tests-debug.patch + ]; }); - # upgrade home-manager until the service-generation fix has landed upstream - # https://github.com/nix-community/home-manager/issues/2846 - home-manager = super.home-manager.overrideAttrs (old: rec { - version = assert super.home-manager.version == "2021-12-25"; "2022-04-08"; - src = self.fetchFromGitHub { - owner = "nix-community"; - repo = "home-manager"; - rev = "f911ebbec927e8e9b582f2e32e2b35f730074cfc"; - sha256 = "07qa2qkbjczj3d0m03jpw85hfj35cbjm48xhifz3viy4khjw88vl"; - }; + evans = super.evans.overrideAttrs (old: { + patches = old.patches or [ ] ++ [ + # add support for unix domain sockets + # https://github.com/ktr0731/evans/pull/680 + ./patches/evans-add-support-for-unix-domain-sockets.patch + ]; }); - python38 = super.python38.override { - packageOverrides = pySelf: pySuper: { - backports-zoneinfo = pySuper.backports-zoneinfo.overridePythonAttrs (_: { - # Outdated test-data, see https://github.com/pganssle/zoneinfo/pull/115 - # Can be dropped when https://github.com/NixOS/nixpkgs/pull/170450 lands. - doCheck = false; - }); - }; - }; + # Imports a patch that fixes usage of this package on versions + # >=1.9. The patch has been proposed upstream, but so far with no + # reactions from the maintainer: + # + # https://github.com/tpm2-software/tpm2-pkcs11/pull/849 + tpm2-pkcs11 = super.tpm2-pkcs11.overrideAttrs (old: { + patches = (old.patches or [ ]) ++ [ ./patches/tpm2-pkcs11-190-dbupgrade.patch ]; + }); } |