diff options
Diffstat (limited to 'third_party/overlays')
26 files changed, 1084 insertions, 0 deletions
diff --git a/third_party/overlays/dhall/OWNERS b/third_party/overlays/dhall/OWNERS new file mode 100644 index 0000000000..a640227914 --- /dev/null +++ b/third_party/overlays/dhall/OWNERS @@ -0,0 +1 @@ +Profpatsch diff --git a/third_party/overlays/dhall/default.nix b/third_party/overlays/dhall/default.nix new file mode 100644 index 0000000000..4625035999 --- /dev/null +++ b/third_party/overlays/dhall/default.nix @@ -0,0 +1,30 @@ +{ ... }: + +self: super: + +let + + # binary releases of dhall tools, since the build in nixpkgs is + # broken most of the time. The binaries are also fully static + # builds, instead of the half-static crap that nixpkgs produces. + easy-dhall-nix = + import + (builtins.fetchTarball { + url = "https://github.com/justinwoo/easy-dhall-nix/archive/dce9acbb99776a7f1344db4751d6080380f76f57.tar.gz"; + sha256 = "0ckp6515gfvbxm08yyll87d9vg8sq2l21gwav2npzvwc3xz2lccf"; + }) + { pkgs = self; }; +in +{ + # ATTN: see the haskell overlay for some overrides we need. + + # dhall = easy-dhall-nix.dhall-simple; + # dhall-nix = easy-dhall-nix.dhall-nix-simple; + dhall-bash = easy-dhall-nix.dhall-bash-simple; + dhall-docs = easy-dhall-nix.dhall-docs-simple; + dhall-json = easy-dhall-nix.dhall-json-simple; + dhall-lsp-server = easy-dhall-nix.dhall-lsp-simple; + # not yet in dhall-simple + # dhall-nixpkgs = easy-dhall-nix.dhall-nixpkgs-simple; + dhall-yaml = easy-dhall-nix.dhall-yaml-simple; +} diff --git a/third_party/overlays/ecl-static.nix b/third_party/overlays/ecl-static.nix new file mode 100644 index 0000000000..d81075bdee --- /dev/null +++ b/third_party/overlays/ecl-static.nix @@ -0,0 +1,28 @@ +{ ... }: + +self: super: + +{ + # Statically linked ECL with statically linked dependencies. + # Works quite well, but solving this properly in a nixpkgs + # context will require figuring out cross compilation (for + # pkgsStatic), so we're gonna use this override for now. + # + # Note that ecl-static does mean that we have things + # statically linked against GMP and ECL which are LGPL. + # I believe this should be alright: The way ppl are gonna + # interact with the distributed binaries (i. e. the binary + # cache) is Nix in the depot monorepo, so the separability + # requirement should be satisfied: Source code or overriding + # would be available as ways to swap out the used GMP in the + # program. + # See https://www.gnu.org/licenses/gpl-faq.en.html#LGPLStaticVsDynamic + ecl-static = (super.pkgsMusl.ecl.override { + inherit (self.pkgsStatic) gmp libffi boehmgc; + }).overrideAttrs (drv: rec { + configureFlags = drv.configureFlags ++ [ + "--disable-shared" + "--with-dffi=no" # will fail at runtime anyways if statically linked + ]; + }); +} diff --git a/third_party/overlays/emacs.nix b/third_party/overlays/emacs.nix new file mode 100644 index 0000000000..341feb5015 --- /dev/null +++ b/third_party/overlays/emacs.nix @@ -0,0 +1,4 @@ +# Emacs overlay from https://github.com/nix-community/emacs-overlay +{ depot, ... }: + +import depot.third_party.sources.emacs-overlay diff --git a/third_party/overlays/haskell/.skip-subtree b/third_party/overlays/haskell/.skip-subtree new file mode 100644 index 0000000000..2a528eaa8a --- /dev/null +++ b/third_party/overlays/haskell/.skip-subtree @@ -0,0 +1 @@ +extra-pkgs need to be callPackage-ed diff --git a/third_party/overlays/haskell/OWNERS b/third_party/overlays/haskell/OWNERS new file mode 100644 index 0000000000..5f87d2f271 --- /dev/null +++ b/third_party/overlays/haskell/OWNERS @@ -0,0 +1,2 @@ +Profpatsch +sterni diff --git a/third_party/overlays/haskell/default.nix b/third_party/overlays/haskell/default.nix new file mode 100644 index 0000000000..5dbb8f45f8 --- /dev/null +++ b/third_party/overlays/haskell/default.nix @@ -0,0 +1,52 @@ +# Defines an overlay for overriding Haskell packages, for example to +# avoid breakage currently present in nixpkgs or to modify package +# versions. + +{ lib, ... }: + +self: super: # overlay parameters for the nixpkgs overlay + +let + haskellLib = self.haskell.lib.compose; +in +{ + haskellPackages = super.haskellPackages.override { + overrides = hsSelf: hsSuper: { + + ihp-hsx = lib.pipe hsSuper.ihp-hsx [ + (haskellLib.overrideSrc { + version = "1.1.0"; + src = "${self.fetchFromGitHub { + owner = "digitallyinduced"; + repo = "ihp"; + rev = "b5d47963c998ccd779aa5c3d46484338fd621f0d"; + sha256 = "sha256-M22W8VX4sRaeU2yVraR0S2t2VOwWGmoteD/M8TahdoE="; + }}/ihp-hsx"; + }) + haskellLib.doJailbreak + ]; + + pa-prelude = hsSelf.callPackage ./extra-pkgs/pa-prelude.nix { }; + pa-error-tree = hsSelf.callPackage ./extra-pkgs/pa-error-tree-0.1.0.0.nix { }; + pa-field-parser = hsSelf.callPackage ./extra-pkgs/pa-field-parser.nix { }; + pa-label = hsSelf.callPackage ./extra-pkgs/pa-label-0.1.0.1.nix { }; + pa-pretty = hsSelf.callPackage ./extra-pkgs/pa-pretty-0.1.1.0.nix { }; + pa-json = hsSelf.callPackage ./extra-pkgs/pa-json.nix { }; + pa-run-command = hsSelf.callPackage ./extra-pkgs/pa-run-command-0.1.0.0.nix { }; + }; + }; + + haskell = lib.recursiveUpdate super.haskell { + packages.ghc8107 = super.haskell.packages.ghc8107.override { + overrides = hsSelf: hsSuper: { + # TODO(sterni): TODO(grfn): patch xanthous to work with random-fu 0.3.*, + # so we can use GHC 9.0.2 and benefit from upstream binary cache. + random-fu = hsSelf.callPackage ./extra-pkgs/random-fu-0.2.nix { }; + rvar = hsSelf.callPackage ./extra-pkgs/rvar-0.2.nix { }; + + # TODO(grfn): port to brick 1.4 (EventM gains an additional type argument in 1.0) + brick = hsSelf.callPackage ./extra-pkgs/brick-0.73.nix { }; + }; + }; + }; +} diff --git a/third_party/overlays/haskell/extra-pkgs/brick-0.73.nix b/third_party/overlays/haskell/extra-pkgs/brick-0.73.nix new file mode 100644 index 0000000000..c5e2883c75 --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/brick-0.73.nix @@ -0,0 +1,70 @@ +{ mkDerivation +, base +, bytestring +, config-ini +, containers +, contravariant +, data-clist +, deepseq +, directory +, dlist +, exceptions +, filepath +, lib +, microlens +, microlens-mtl +, microlens-th +, QuickCheck +, stm +, template-haskell +, text +, text-zipper +, transformers +, unix +, vector +, vty +, word-wrap +}: +mkDerivation { + pname = "brick"; + version = "0.73"; + sha256 = "741c8d0717f0ab5addd5d3acc88cb36d645a0c73907bde509b2fd9d9bc02039c"; + isLibrary = true; + isExecutable = true; + libraryHaskellDepends = [ + base + bytestring + config-ini + containers + contravariant + data-clist + deepseq + directory + dlist + exceptions + filepath + microlens + microlens-mtl + microlens-th + stm + template-haskell + text + text-zipper + transformers + unix + vector + vty + word-wrap + ]; + testHaskellDepends = [ + base + containers + microlens + QuickCheck + vector + vty + ]; + homepage = "https://github.com/jtdaugherty/brick/"; + description = "A declarative terminal user interface library"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/pa-error-tree-0.1.0.0.nix b/third_party/overlays/haskell/extra-pkgs/pa-error-tree-0.1.0.0.nix new file mode 100644 index 0000000000..a38cd4efaa --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/pa-error-tree-0.1.0.0.nix @@ -0,0 +1,10 @@ +{ mkDerivation, base, containers, lib, pa-prelude }: +mkDerivation { + pname = "pa-error-tree"; + version = "0.1.0.0"; + sha256 = "f82d3d905e8d9f0d31c81f31c424b9a95c65a8925517ccac92134f410cf8d639"; + libraryHaskellDepends = [ base containers pa-prelude ]; + homepage = "https://github.com/possehl-analytics/pa-hackage"; + description = "Collect a tree of errors and pretty-print"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/pa-field-parser.nix b/third_party/overlays/haskell/extra-pkgs/pa-field-parser.nix new file mode 100644 index 0000000000..a3c146ee09 --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/pa-field-parser.nix @@ -0,0 +1,39 @@ +{ mkDerivation +, aeson +, aeson-better-errors +, attoparsec +, base +, case-insensitive +, containers +, lib +, pa-error-tree +, pa-prelude +, scientific +, semigroupoids +, template-haskell +, text +, time +}: +mkDerivation { + pname = "pa-field-parser"; + version = "0.3.0.0"; + sha256 = "528c2b6bf5ad6454861b059c7eb6924f4c32bcb5b8faa4c2389d9ddfd92fcd57"; + libraryHaskellDepends = [ + aeson + aeson-better-errors + attoparsec + base + case-insensitive + containers + pa-error-tree + pa-prelude + scientific + semigroupoids + template-haskell + text + time + ]; + homepage = "https://github.com/possehl-analytics/pa-hackage"; + description = "“Vertical” parsing of values"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/pa-json.nix b/third_party/overlays/haskell/extra-pkgs/pa-json.nix new file mode 100644 index 0000000000..8ce838b22c --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/pa-json.nix @@ -0,0 +1,43 @@ +{ mkDerivation +, aeson +, aeson-better-errors +, aeson-pretty +, base +, base64-bytestring +, bytestring +, containers +, lib +, pa-error-tree +, pa-field-parser +, pa-label +, pa-prelude +, scientific +, text +, time +, vector +}: +mkDerivation { + pname = "pa-json"; + version = "0.3.0.0"; + sha256 = "45e79765e57e21400f3f3b1e86094473fac61d298618d7e34f6cad4988d8923b"; + libraryHaskellDepends = [ + aeson + aeson-better-errors + aeson-pretty + base + base64-bytestring + bytestring + containers + pa-error-tree + pa-field-parser + pa-label + pa-prelude + scientific + text + time + vector + ]; + homepage = "https://github.com/possehl-analytics/pa-hackage"; + description = "Our JSON parsers/encoders"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/pa-label-0.1.0.1.nix b/third_party/overlays/haskell/extra-pkgs/pa-label-0.1.0.1.nix new file mode 100644 index 0000000000..1da78260cc --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/pa-label-0.1.0.1.nix @@ -0,0 +1,10 @@ +{ mkDerivation, base, lib }: +mkDerivation { + pname = "pa-label"; + version = "0.1.0.1"; + sha256 = "0131ab7718d910a94cd8cc881e51b7371a060dadfeabc8fd78513a7f27ee8d35"; + libraryHaskellDepends = [ base ]; + homepage = "https://github.com/possehl-analytics/pa-hackage"; + description = "Labels, and labelled tuples and enums (GHC >9.2)"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/pa-prelude.nix b/third_party/overlays/haskell/extra-pkgs/pa-prelude.nix new file mode 100644 index 0000000000..17e1996ab6 --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/pa-prelude.nix @@ -0,0 +1,43 @@ +{ mkDerivation +, base +, bytestring +, containers +, error +, exceptions +, lib +, mtl +, profunctors +, PyF +, scientific +, semigroupoids +, template-haskell +, text +, these +, validation-selective +, vector +}: +mkDerivation { + pname = "pa-prelude"; + version = "0.2.0.0"; + sha256 = "68015f7c19e9c618fc04e2516baccfce52af24efb9ca1480162c9ea0aef7f301"; + libraryHaskellDepends = [ + base + bytestring + containers + error + exceptions + mtl + profunctors + PyF + scientific + semigroupoids + template-haskell + text + these + validation-selective + vector + ]; + homepage = "https://github.com/possehl-analytics/pa-hackage"; + description = "The Possehl Analytics Prelude"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/pa-pretty-0.1.1.0.nix b/third_party/overlays/haskell/extra-pkgs/pa-pretty-0.1.1.0.nix new file mode 100644 index 0000000000..d6dadef849 --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/pa-pretty-0.1.1.0.nix @@ -0,0 +1,29 @@ +{ mkDerivation +, aeson +, aeson-pretty +, ansi-terminal +, base +, hscolour +, lib +, nicify-lib +, pa-prelude +, text +}: +mkDerivation { + pname = "pa-pretty"; + version = "0.1.1.0"; + sha256 = "da925a7cf2ac49c5769d7ebd08c2599b537efe45b3d506bf4d7c8673633ef6c9"; + libraryHaskellDepends = [ + aeson + aeson-pretty + ansi-terminal + base + hscolour + nicify-lib + pa-prelude + text + ]; + homepage = "https://github.com/possehl-analytics/pa-hackage"; + description = "Some pretty-printing helpers"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/pa-run-command-0.1.0.0.nix b/third_party/overlays/haskell/extra-pkgs/pa-run-command-0.1.0.0.nix new file mode 100644 index 0000000000..b12eb5efbf --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/pa-run-command-0.1.0.0.nix @@ -0,0 +1,25 @@ +{ mkDerivation +, base +, bytestring +, lib +, monad-logger +, pa-prelude +, text +, typed-process +}: +mkDerivation { + pname = "pa-run-command"; + version = "0.1.0.0"; + sha256 = "37837e0cddedc9b615063f0357115739c53b5dcb8af82ce86a95a3a5c88c29a3"; + libraryHaskellDepends = [ + base + bytestring + monad-logger + pa-prelude + text + typed-process + ]; + homepage = "https://github.com/possehl-analytics/pa-hackage"; + description = "Helper functions for spawning subprocesses"; + license = lib.licenses.bsd3; +} diff --git a/third_party/overlays/haskell/extra-pkgs/random-fu-0.2.nix b/third_party/overlays/haskell/extra-pkgs/random-fu-0.2.nix new file mode 100644 index 0000000000..1626eca7be --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/random-fu-0.2.nix @@ -0,0 +1,41 @@ +{ mkDerivation +, base +, erf +, lib +, math-functions +, monad-loops +, mtl +, random +, random-shuffle +, random-source +, rvar +, syb +, template-haskell +, transformers +, vector +}: +mkDerivation { + pname = "random-fu"; + version = "0.2.7.7"; + sha256 = "8466bcfb5290bdc30a571c91e1eb526c419ea9773bc118996778b516cfc665ca"; + revision = "1"; + editedCabalFile = "16nhymfriygqr2by9v72vdzv93v6vhd9z07pgaji4zvv66jikv82"; + libraryHaskellDepends = [ + base + erf + math-functions + monad-loops + mtl + random + random-shuffle + random-source + rvar + syb + template-haskell + transformers + vector + ]; + homepage = "https://github.com/mokus0/random-fu"; + description = "Random number generation"; + license = lib.licenses.publicDomain; +} diff --git a/third_party/overlays/haskell/extra-pkgs/rvar-0.2.nix b/third_party/overlays/haskell/extra-pkgs/rvar-0.2.nix new file mode 100644 index 0000000000..c00f5a1a8d --- /dev/null +++ b/third_party/overlays/haskell/extra-pkgs/rvar-0.2.nix @@ -0,0 +1,25 @@ +{ mkDerivation +, base +, lib +, MonadPrompt +, mtl +, random-source +, transformers +}: +mkDerivation { + pname = "rvar"; + version = "0.2.0.6"; + sha256 = "01e18875ffde43f9591a8acd9f60c9c51704a026e51c1a6797faecd1c7ae8cd3"; + revision = "1"; + editedCabalFile = "1jn9ivlj3k65n8d9sfsp882m5lvni1ah79mk0cvkz91pgywvkiyq"; + libraryHaskellDepends = [ + base + MonadPrompt + mtl + random-source + transformers + ]; + homepage = "https://github.com/mokus0/random-fu"; + description = "Random Variables"; + license = lib.licenses.publicDomain; +} diff --git a/third_party/overlays/patches/.skip-tree b/third_party/overlays/patches/.skip-tree new file mode 100644 index 0000000000..86eae51a6d --- /dev/null +++ b/third_party/overlays/patches/.skip-tree @@ -0,0 +1 @@ +No readTree-compatible files. diff --git a/third_party/overlays/patches/0001-configure-ac-version.patch b/third_party/overlays/patches/0001-configure-ac-version.patch new file mode 100644 index 0000000000..fa2575cb93 --- /dev/null +++ b/third_party/overlays/patches/0001-configure-ac-version.patch @@ -0,0 +1,13 @@ +diff --git a/configure.ac b/configure.ac +index e861e42..018c19c 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -26,7 +26,7 @@ + #;**********************************************************************; + + AC_INIT([tpm2-pkcs11], +- [m4_esyscmd_s([git describe --tags --always --dirty])], ++ [git-@VERSION@], + [https://github.com/tpm2-software/tpm2-pkcs11/issues], + [], + [https://github.com/tpm2-software/tpm2-pkcs11]) diff --git a/third_party/overlays/patches/buf-tests-dont-use-file-transport.patch b/third_party/overlays/patches/buf-tests-dont-use-file-transport.patch new file mode 100644 index 0000000000..34be80eb36 --- /dev/null +++ b/third_party/overlays/patches/buf-tests-dont-use-file-transport.patch @@ -0,0 +1,64 @@ +commit e9219b88de5ed37af337ee2d2e71e7ec7c0aad1b +Author: Robbert van Ginkel <rvanginkel@buf.build> +Date: Thu Oct 20 16:43:28 2022 -0400 + + Fix git unit test by using fake git server rather than file:// (#1518) + + More recent versions of git fix a CVE by disabling some usage of the + `file://` transport, see + https://github.blog/2022-10-18-git-security-vulnerabilities-announced/#cve-2022-39253. + We were using this transport in tests. + + Instead, use https://git-scm.com/docs/git-http-backend to serve up this + repository locally so we don't have to use the file protocol. This + should be a more accurate tests, since we mostly expect submodules to + come from servers. + +diff --git a/.golangci.yml b/.golangci.yml +index 318d1171..865e03e7 100644 +--- a/.golangci.yml ++++ b/.golangci.yml +@@ -136,3 +136,8 @@ issues: + - linters: + - containedctx + path: private/bufpkg/bufmodule/bufmoduleprotocompile ++ # We should be able to use net/http/cgi in a unit test, in addition the CVE mentions only versions of go < 1.6.3 are affected. ++ - linters: ++ - gosec ++ path: private/pkg/git/git_test.go ++ text: "G504:" +diff --git a/private/pkg/git/git_test.go b/private/pkg/git/git_test.go +index 7b77b6cd..7132054e 100644 +--- a/private/pkg/git/git_test.go ++++ b/private/pkg/git/git_test.go +@@ -17,6 +17,8 @@ package git + import ( + "context" + "errors" ++ "net/http/cgi" ++ "net/http/httptest" + "os" + "os/exec" + "path/filepath" +@@ -213,6 +215,21 @@ func createGitDirs( + runCommand(ctx, t, container, runner, "git", "-C", submodulePath, "add", "test.proto") + runCommand(ctx, t, container, runner, "git", "-C", submodulePath, "commit", "-m", "commit 0") + ++ gitExecPath, err := command.RunStdout(ctx, container, runner, "git", "--exec-path") ++ require.NoError(t, err) ++ t.Log(filepath.Join(string(gitExecPath), "git-http-backend")) ++ // https://git-scm.com/docs/git-http-backend#_description ++ f, err := os.Create(filepath.Join(submodulePath, ".git", "git-daemon-export-ok")) ++ require.NoError(t, err) ++ require.NoError(t, f.Close()) ++ server := httptest.NewServer(&cgi.Handler{ ++ Path: filepath.Join(strings.TrimSpace(string(gitExecPath)), "git-http-backend"), ++ Dir: submodulePath, ++ Env: []string{"GIT_PROJECT_ROOT=" + submodulePath}, ++ }) ++ t.Cleanup(server.Close) ++ submodulePath = server.URL ++ + originPath := filepath.Join(tmpDir, "origin") + require.NoError(t, os.MkdirAll(originPath, 0777)) + runCommand(ctx, t, container, runner, "git", "-C", originPath, "init") diff --git a/third_party/overlays/patches/clickhouse-support-reading-arrow-LargeListArray.patch b/third_party/overlays/patches/clickhouse-support-reading-arrow-LargeListArray.patch new file mode 100644 index 0000000000..9e79aa7267 --- /dev/null +++ b/third_party/overlays/patches/clickhouse-support-reading-arrow-LargeListArray.patch @@ -0,0 +1,106 @@ +From cdea2e8ad98995202ce81c9c030f2ae64d73b05a Mon Sep 17 00:00:00 2001 +From: edef <edef@edef.eu> +Date: Mon, 30 Oct 2023 08:08:10 +0000 +Subject: [PATCH] Support reading arrow::LargeListArray + +--- + .../Formats/Impl/ArrowColumnToCHColumn.cpp | 33 +++++++++++++++---- + 1 file changed, 26 insertions(+), 7 deletions(-) + +diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +index 6f9d49498f2..b93846cd4eb 100644 +--- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp ++++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +@@ -436,6 +436,22 @@ static ColumnPtr readByteMapFromArrowColumn(std::shared_ptr<arrow::ChunkedArray> + return nullmap_column; + } + ++template <typename T> ++struct ArrowOffsetArray; ++ ++template <> ++struct ArrowOffsetArray<arrow::ListArray> ++{ ++ using type = arrow::Int32Array; ++}; ++ ++template <> ++struct ArrowOffsetArray<arrow::LargeListArray> ++{ ++ using type = arrow::Int64Array; ++}; ++ ++template <typename ArrowListArray> + static ColumnPtr readOffsetsFromArrowListColumn(std::shared_ptr<arrow::ChunkedArray> & arrow_column) + { + auto offsets_column = ColumnUInt64::create(); +@@ -444,9 +460,9 @@ static ColumnPtr readOffsetsFromArrowListColumn(std::shared_ptr<arrow::ChunkedAr + + for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i) + { +- arrow::ListArray & list_chunk = dynamic_cast<arrow::ListArray &>(*(arrow_column->chunk(chunk_i))); ++ ArrowListArray & list_chunk = dynamic_cast<ArrowListArray &>(*(arrow_column->chunk(chunk_i))); + auto arrow_offsets_array = list_chunk.offsets(); +- auto & arrow_offsets = dynamic_cast<arrow::Int32Array &>(*arrow_offsets_array); ++ auto & arrow_offsets = dynamic_cast<ArrowOffsetArray<ArrowListArray>::type &>(*arrow_offsets_array); + + /* + * CH uses element size as "offsets", while arrow uses actual offsets as offsets. +@@ -602,13 +618,14 @@ static ColumnPtr readColumnWithIndexesData(std::shared_ptr<arrow::ChunkedArray> + } + } + ++template <typename ArrowListArray> + static std::shared_ptr<arrow::ChunkedArray> getNestedArrowColumn(std::shared_ptr<arrow::ChunkedArray> & arrow_column) + { + arrow::ArrayVector array_vector; + array_vector.reserve(arrow_column->num_chunks()); + for (int chunk_i = 0, num_chunks = arrow_column->num_chunks(); chunk_i < num_chunks; ++chunk_i) + { +- arrow::ListArray & list_chunk = dynamic_cast<arrow::ListArray &>(*(arrow_column->chunk(chunk_i))); ++ ArrowListArray & list_chunk = dynamic_cast<ArrowListArray &>(*(arrow_column->chunk(chunk_i))); + + /* + * It seems like arrow::ListArray::values() (nested column data) might or might not be shared across chunks. +@@ -819,12 +836,12 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( + key_type_hint = map_type_hint->getKeyType(); + } + } +- auto arrow_nested_column = getNestedArrowColumn(arrow_column); ++ auto arrow_nested_column = getNestedArrowColumn<arrow::ListArray>(arrow_column); + auto nested_column = readColumnFromArrowColumn(arrow_nested_column, column_name, format_name, false, dictionary_infos, allow_null_type, skip_columns_with_unsupported_types, skipped, date_time_overflow_behavior, nested_type_hint, true); + if (skipped) + return {}; + +- auto offsets_column = readOffsetsFromArrowListColumn(arrow_column); ++ auto offsets_column = readOffsetsFromArrowListColumn<arrow::ListArray>(arrow_column); + + const auto * tuple_column = assert_cast<const ColumnTuple *>(nested_column.column.get()); + const auto * tuple_type = assert_cast<const DataTypeTuple *>(nested_column.type.get()); +@@ -846,7 +863,9 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( + return {std::move(map_column), std::move(map_type), column_name}; + } + case arrow::Type::LIST: ++ case arrow::Type::LARGE_LIST: + { ++ bool is_large = arrow_column->type()->id() == arrow::Type::LARGE_LIST; + DataTypePtr nested_type_hint; + if (type_hint) + { +@@ -854,11 +873,11 @@ static ColumnWithTypeAndName readColumnFromArrowColumn( + if (array_type_hint) + nested_type_hint = array_type_hint->getNestedType(); + } +- auto arrow_nested_column = getNestedArrowColumn(arrow_column); ++ auto arrow_nested_column = is_large ? getNestedArrowColumn<arrow::LargeListArray>(arrow_column) : getNestedArrowColumn<arrow::ListArray>(arrow_column); + auto nested_column = readColumnFromArrowColumn(arrow_nested_column, column_name, format_name, false, dictionary_infos, allow_null_type, skip_columns_with_unsupported_types, skipped, date_time_overflow_behavior, nested_type_hint); + if (skipped) + return {}; +- auto offsets_column = readOffsetsFromArrowListColumn(arrow_column); ++ auto offsets_column = is_large ? readOffsetsFromArrowListColumn<arrow::LargeListArray>(arrow_column) : readOffsetsFromArrowListColumn<arrow::ListArray>(arrow_column); + auto array_column = ColumnArray::create(nested_column.column, offsets_column); + auto array_type = std::make_shared<DataTypeArray>(nested_column.type); + return {std::move(array_column), std::move(array_type), column_name}; +-- +2.42.0 + diff --git a/third_party/overlays/patches/crate2nix-run-tests-in-build-source.patch b/third_party/overlays/patches/crate2nix-run-tests-in-build-source.patch new file mode 100644 index 0000000000..52793270e6 --- /dev/null +++ b/third_party/overlays/patches/crate2nix-run-tests-in-build-source.patch @@ -0,0 +1,69 @@ +From 7cf084f73f7d15fe0538a625182fa7179c083b3d Mon Sep 17 00:00:00 2001 +From: Raito Bezarius <masterancpp@gmail.com> +Date: Tue, 16 Jan 2024 02:10:48 +0100 +Subject: [PATCH] fix(template): run tests in `/build/source` instead `/build` + +Previously, the source tree was located inline in `/build` during tests, this was a mistake +because the crates more than often are built in `/build/source` as per the `sourceRoot` system. + +This can cause issues with test binaries hardcoding `/build/source/...` as their choice for doing things, +causing them to be confused in the test phase which is relocated without rewriting the paths inside test binaries. + +We fix that by relocating ourselves in the right hierarchy. + +This is a "simple" fix in the sense that more edge cases could exist but they are hard to reason about +because they would be crates using custom `sourceRoot`, i.e. having `crate.sourceRoot` set and then it becomes +a bit hard to reproduce the hierarchy, you need to analyze whether the path is absolute or relative, + +If it's relative, you can just reuse it and reproduce that specific hierarchy. +If it's absolute, you need to cut the "absolute" meaningless part, e.g. `$NIX_BUILD_TOP/` and proceed like +it's a relative path IMHO. +--- + crate2nix/Cargo.nix | 10 ++++++++++ + crate2nix/templates/nix/crate2nix/default.nix | 10 ++++++++++ + +diff --git a/Cargo.nix b/Cargo.nix +index 6ef7a49..172ff34 100644 +--- a/Cargo.nix ++++ b/Cargo.nix +@@ -2889,6 +2889,16 @@ rec { + # recreate a file hierarchy as when running tests with cargo + + # the source for test data ++ # It's necessary to locate the source in $NIX_BUILD_TOP/source/ ++ # instead of $NIX_BUILD_TOP/ ++ # because we compiled those test binaries in the former and not the latter. ++ # So all paths will expect source tree to be there and not in the build top directly. ++ # For example: $NIX_BUILD_TOP := /build in general, if you ask yourself. ++ # TODO(raitobezarius): I believe there could be more edge cases if `crate.sourceRoot` ++ # do exist but it's very hard to reason about them, so let's wait until the first bug report. ++ mkdir -p source/ ++ cd source/ ++ + ${pkgs.buildPackages.xorg.lndir}/bin/lndir ${crate.src} + + # build outputs +diff --git a/crate2nix/templates/nix/crate2nix/default.nix b/crate2nix/templates/nix/crate2nix/default.nix +index e4fc2e9..dfb14c4 100644 +--- a/templates/nix/crate2nix/default.nix ++++ b/templates/nix/crate2nix/default.nix +@@ -135,6 +135,16 @@ rec { + # recreate a file hierarchy as when running tests with cargo + + # the source for test data ++ # It's necessary to locate the source in $NIX_BUILD_TOP/source/ ++ # instead of $NIX_BUILD_TOP/ ++ # because we compiled those test binaries in the former and not the latter. ++ # So all paths will expect source tree to be there and not in the build top directly. ++ # For example: $NIX_BUILD_TOP := /build in general, if you ask yourself. ++ # TODO(raitobezarius): I believe there could be more edge cases if `crate.sourceRoot` ++ # do exist but it's very hard to reason about them, so let's wait until the first bug report. ++ mkdir -p source/ ++ cd source/ ++ + ${pkgs.buildPackages.xorg.lndir}/bin/lndir ${crate.src} + + # build outputs +-- +2.43.0 + diff --git a/third_party/overlays/patches/evans-add-support-for-unix-domain-sockets.patch b/third_party/overlays/patches/evans-add-support-for-unix-domain-sockets.patch new file mode 100644 index 0000000000..c66528f538 --- /dev/null +++ b/third_party/overlays/patches/evans-add-support-for-unix-domain-sockets.patch @@ -0,0 +1,39 @@ +From 55d7e7af7c56f678eb817059417241bb61ee5181 Mon Sep 17 00:00:00 2001 +From: Florian Klink <flokli@flokli.de> +Date: Sun, 8 Oct 2023 11:00:27 +0200 +Subject: [PATCH] add support for unix domain sockets + +grpc.NewClient already supports connecting to unix domain sockets, and +accepts a string anyways. + +As a quick fix, detect the `address` starting with `unix://` and don't +add the port. + +In the long term, we might want to deprecate `host` and `port` cmdline +args in favor of a single `address` arg. +--- + mode/common.go | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/mode/common.go b/mode/common.go +index dfc7839..55f1e36 100644 +--- a/mode/common.go ++++ b/mode/common.go +@@ -13,7 +13,13 @@ import ( + ) + + func newGRPCClient(cfg *config.Config) (grpc.Client, error) { +- addr := fmt.Sprintf("%s:%s", cfg.Server.Host, cfg.Server.Port) ++ addr := cfg.Server.Host ++ ++ // as long as the address doesn't start with unix, also add the port. ++ if !strings.HasPrefix(cfg.Server.Host, "unix://") { ++ addr = fmt.Sprintf("%s:%s", cfg.Server.Host, cfg.Server.Port) ++ } ++ + if cfg.Request.Web { + //TODO: remove second arg + return grpc.NewWebClient(addr, cfg.Server.Reflection, false, "", "", "", grpc.Headers(cfg.Request.Header)), nil +-- +2.42.0 + diff --git a/third_party/overlays/patches/notmuch-dottime.patch b/third_party/overlays/patches/notmuch-dottime.patch new file mode 100644 index 0000000000..7a9cfc6cc2 --- /dev/null +++ b/third_party/overlays/patches/notmuch-dottime.patch @@ -0,0 +1,81 @@ +From 569438172fa0e38129de4e61a72e06eff3330dca Mon Sep 17 00:00:00 2001 +From: Vincent Ambo <tazjin@google.com> +Date: Thu, 10 Dec 2020 10:53:47 +0100 +Subject: [PATCH] time: Use dottime for formatting non-relative timestamps + +--- + notmuch-time.c | 10 +++++----- + util/gmime-extra.c | 7 +++++-- + util/gmime-extra.h | 2 ++ + 3 files changed, 12 insertions(+), 7 deletions(-) + +diff --git a/notmuch-time.c b/notmuch-time.c +index cc7ffc23..3030a667 100644 +--- a/notmuch-time.c ++++ b/notmuch-time.c +@@ -50,8 +50,8 @@ notmuch_time_relative_date (const void *ctx, time_t then) + time_t delta; + char *result; + +- localtime_r (&now, &tm_now); +- localtime_r (&then, &tm_then); ++ gmtime_r (&now, &tm_now); ++ gmtime_r (&then, &tm_then); + + result = talloc_zero_size (ctx, RELATIVE_DATE_MAX); + if (result == NULL) +@@ -78,16 +78,16 @@ notmuch_time_relative_date (const void *ctx, time_t then) + if (tm_then.tm_wday == tm_now.tm_wday && + delta < DAY) { + strftime (result, RELATIVE_DATE_MAX, +- "Today %R", &tm_then); /* Today 12:30 */ ++ "Today %k·%M", &tm_then); /* Today 12·30 */ + return result; + } else if ((tm_now.tm_wday + 7 - tm_then.tm_wday) % 7 == 1) { + strftime (result, RELATIVE_DATE_MAX, +- "Yest. %R", &tm_then); /* Yest. 12:30 */ ++ "Yest. %k·%M", &tm_then); /* Yest. 12·30 */ + return result; + } else { + if (tm_then.tm_wday != tm_now.tm_wday) { + strftime (result, RELATIVE_DATE_MAX, +- "%a. %R", &tm_then); /* Mon. 12:30 */ ++ "%a. %k·%M", &tm_then); /* Mon. 12·30 */ + return result; + } + } +diff --git a/util/gmime-extra.c b/util/gmime-extra.c +index 04d8ed3d..868a2f69 100644 +--- a/util/gmime-extra.c ++++ b/util/gmime-extra.c +@@ -131,10 +131,13 @@ g_mime_message_get_date_string (void *ctx, GMimeMessage *message) + GDateTime *parsed_date = g_mime_message_get_date (message); + + if (parsed_date) { +- char *date = g_mime_utils_header_format_date (parsed_date); ++ char *date = g_date_time_format( ++ parsed_date, ++ "%a, %d %b %Y %H·%M%z" ++ ); + return g_string_talloc_strdup (ctx, date); + } else { +- return talloc_strdup (ctx, "Thu, 01 Jan 1970 00:00:00 +0000"); ++ return talloc_strdup (ctx, "Thu, 01 Jan 1970 00·00:00"); + } + } + +diff --git a/util/gmime-extra.h b/util/gmime-extra.h +index 094309ec..e6c98f8d 100644 +--- a/util/gmime-extra.h ++++ b/util/gmime-extra.h +@@ -1,5 +1,7 @@ + #ifndef _GMIME_EXTRA_H + #define _GMIME_EXTRA_H ++#include <glib.h> ++#include <glib/gprintf.h> + #include <gmime/gmime.h> + #include <talloc.h> + +-- +2.29.2.576.ga3fc446d84-goog + diff --git a/third_party/overlays/patches/tpm2-pkcs11.nix b/third_party/overlays/patches/tpm2-pkcs11.nix new file mode 100644 index 0000000000..2e7db7aca3 --- /dev/null +++ b/third_party/overlays/patches/tpm2-pkcs11.nix @@ -0,0 +1,105 @@ +{ stdenv +, lib +, fetchFromGitHub +, substituteAll +, pkg-config +, autoreconfHook +, autoconf-archive +, makeWrapper +, patchelf +, tpm2-tss +, tpm2-tools +, opensc +, openssl +, sqlite +, python3 +, glibc +, libyaml +, abrmdSupport ? true +, tpm2-abrmd ? null +}: + +stdenv.mkDerivation rec { + pname = "tpm2-pkcs11"; + version = "1.8.0"; + + src = fetchFromGitHub { + owner = "tpm2-software"; + repo = pname; + rev = version; + sha256 = "sha256-f5wi0nIM071yaQCwPkY1agKc7OEQa/IxHJc4V2i0Q9I="; + }; + + patches = lib.singleton ( + substituteAll { + src = ./0001-configure-ac-version.patch; + VERSION = version; + }); + + # The preConfigure phase doesn't seem to be working here + # ./bootstrap MUST be executed as the first step, before all + # of the autoreconfHook stuff + postPatch = '' + ./bootstrap + ''; + + nativeBuildInputs = [ + pkg-config + autoreconfHook + autoconf-archive + makeWrapper + patchelf + ]; + buildInputs = [ + tpm2-tss + tpm2-tools + opensc + openssl + sqlite + libyaml + (python3.withPackages (ps: with ps; [ packaging pyyaml cryptography pyasn1-modules tpm2-pytss ])) + ]; + + outputs = [ "out" "bin" "dev" ]; + + dontStrip = true; + dontPatchELF = true; + + # To be able to use the userspace resource manager, the RUNPATH must + # explicitly include the tpm2-abrmd shared libraries. + preFixup = + let + rpath = lib.makeLibraryPath ( + (lib.optional abrmdSupport tpm2-abrmd) + ++ [ + tpm2-tss + sqlite + openssl + glibc + libyaml + ] + ); + in + '' + patchelf \ + --set-rpath ${rpath} \ + ${lib.optionalString abrmdSupport "--add-needed ${lib.makeLibraryPath [tpm2-abrmd]}/libtss2-tcti-tabrmd.so"} \ + --add-needed ${lib.makeLibraryPath [tpm2-tss]}/libtss2-tcti-device.so \ + $out/lib/libtpm2_pkcs11.so.0.0.0 + ''; + + postInstall = '' + mkdir -p $bin/bin/ $bin/share/tpm2_pkcs11/ + mv ./tools/* $bin/share/tpm2_pkcs11/ + makeWrapper $bin/share/tpm2_pkcs11/tpm2_ptool.py $bin/bin/tpm2_ptool \ + --prefix PATH : ${lib.makeBinPath [ tpm2-tools ]} + ''; + + meta = with lib; { + description = "A PKCS#11 interface for TPM2 hardware"; + homepage = "https://github.com/tpm2-software/tpm2-pkcs11"; + license = licenses.bsd2; + platforms = platforms.linux; + maintainers = with maintainers; [ matthiasbeyer ]; + }; +} diff --git a/third_party/overlays/tvl.nix b/third_party/overlays/tvl.nix new file mode 100644 index 0000000000..861c66694a --- /dev/null +++ b/third_party/overlays/tvl.nix @@ -0,0 +1,153 @@ +# This overlay is used to make TVL-specific modifications in the +# nixpkgs tree, where required. +{ lib, depot, localSystem, ... }: + +self: super: +depot.nix.readTree.drvTargets { + nix_2_3 = (super.nix_2_3.override { + # flaky tests, long painful build, see https://github.com/NixOS/nixpkgs/pull/266443 + withAWS = false; + }); + nix = self.nix_2_3; + nix_latest = super.nix.override ({ + # flaky tests, long painful build, see https://github.com/NixOS/nixpkgs/pull/266443 + withAWS = false; + }); + + # To match telega in emacs-overlay or wherever + tdlib = super.tdlib.overrideAttrs (_: { + version = "1.8.23"; + src = self.fetchFromGitHub { + owner = "tdlib"; + repo = "td"; + rev = "27c3eaeb4964bd5f18d8488e354abde1a4383e49"; + sha256 = "14f65dfmg2p5hyvi3lffvvazwcd3i3jrrw3c2pwrc5yfgxk3662g"; + }; + }); + + home-manager = super.home-manager.overrideAttrs (_: { + src = depot.third_party.sources.home-manager; + version = "git-" + + builtins.substring 0 7 depot.third_party.sources.home-manager.rev; + }); + + # Add our Emacs packages to the fixpoint + emacsPackagesFor = emacs: ( + (super.emacsPackagesFor emacs).overrideScope' (eself: esuper: { + tvlPackages = depot.tools.emacs-pkgs // depot.third_party.emacs; + + # Use the notmuch from nixpkgs instead of from the Emacs + # overlay, to avoid versions being out of sync. + notmuch = super.notmuch.emacs; + + # Build EXWM with the depot sources instead. + depotExwm = eself.callPackage depot.third_party.exwm.override { }; + + # Workaround for magit checking the git version at load time + magit = esuper.magit.overrideAttrs (_: { + propagatedNativeBuildInputs = [ + self.git + ]; + }); + + # Pin xelb to a newer one until the new maintainers do a release. + xelb = eself.trivialBuild { + pname = "xelb"; + version = "0.19-dev"; # invented version, last actual release was 0.18 + + src = self.fetchFromGitHub { + owner = "emacs-exwm"; + repo = "xelb"; + rev = "86089eba2de6c818bfa2fac075cb7ad876262798"; + sha256 = "1mmlrd2zpcwiv8gh10y7lrpflnbmsycdascrxjr3bfcwa8yx7901"; + }; + }; + }) + ); + + # dottime support for notmuch + notmuch = super.notmuch.overrideAttrs (old: { + passthru = old.passthru // { + patches = old.patches ++ [ ./patches/notmuch-dottime.patch ]; + }; + }); + + # nix-serve does not work with nix 2.4 + # https://github.com/edolstra/nix-serve/issues/28 + nix-serve = super.nix-serve.override { nix = self.nix_2_3; }; + + # Avoid builds of mkShell derivations in CI. + mkShell = super.lib.makeOverridable (args: (super.mkShell args).overrideAttrs (_: { + passthru = { + meta.ci.skip = true; + }; + })); + + crate2nix = super.rustPlatform.buildRustPackage rec { + pname = "crate2nix"; + version = "0.13.0"; + + src = super.fetchFromGitHub { + owner = "nix-community"; + repo = "crate2nix"; + rev = "ceb06eb7e76afb9e01a5f069aae136f97df72730"; + hash = "sha256-JTMe8GViCQt51WUiaaoIPmWtwEeeYrl6pBxo2DNuKig="; + }; + + patches = [ ./patches/crate2nix-run-tests-in-build-source.patch ]; + + sourceRoot = "${src.name}/crate2nix"; + + cargoHash = "sha256-dhlSXY1CJE+JJt+6Y7W1MVMz36nwr6ny543py1TcjyY="; + + nativeBuildInputs = [ super.makeWrapper ]; + + # Tests use nix(1), which tries (and fails) to set up /nix/var inside the + # sandbox + doCheck = false; + + postFixup = '' + wrapProgram $out/bin/crate2nix \ + --suffix PATH ":" ${lib.makeBinPath (with self; [ cargo nix_latest nix-prefetch-git ])} + + rm -rf $out/lib $out/bin/crate2nix.d + mkdir -p \ + $out/share/bash-completion/completions \ + $out/share/zsh/vendor-completions + $out/bin/crate2nix completions -s 'bash' -o $out/share/bash-completion/completions + $out/bin/crate2nix completions -s 'zsh' -o $out/share/zsh/vendor-completions + ''; + }; + + evans = super.evans.overrideAttrs (old: { + patches = old.patches or [ ] ++ [ + # add support for unix domain sockets + # https://github.com/ktr0731/evans/pull/680 + ./patches/evans-add-support-for-unix-domain-sockets.patch + ]; + }); + + # Package gerrit-queue, which is not in nixpkgs yet + gerrit-queue = super.buildGoModule { + pname = "gerrit-queue"; + version = "unstable-2023-10-20"; + vendorHash = "sha256-+Ig4D46NphzpWKXO23Haea9EqVtpda8v9zLPJkbe3bQ="; + src = super.fetchFromGitHub { + owner = "flokli"; + repo = "gerrit-queue"; + rev = "0186dbde15c9b11dc17b422feb74c842f6fa605a"; + hash = "sha256-zXB5vre/Vr7UOyeMnf2RCtMKm+v5RENH7kGPr/2o7mI="; + }; + + meta = with lib; { + description = "Gerrit submit bot"; + homepage = "https://github.com/tweag/gerrit-queue"; + license = licenses.asl20; + }; + }; + + # OpenVPN + TPM2 is broken on versions of this package somewhere + # after 1.8.0, but it is a critical dependency for tazjin. For this + # reason it is vendored from a specific nixpkgs commit. + tpm2-pkcs11 = self.callPackage ./patches/tpm2-pkcs11.nix { }; +} |